Unicode-compliant islower/uppercase (JuliaLang#38574)

* Unicode-compliant islower/uppercase

* don't test isletter for non-L* letters

* include titlecase in alphas test

* add news

NEWS.md

@@ -36,6 +36,7 @@ New library features
 Standard library changes
 ------------------------
 
+* `islowercase` and `isuppercase` are now compliant with the Unicode lower/uppercase categories ([#38574]).
 
 #### Package Manager
 

base/strings/unicode.jl

@@ -280,9 +280,8 @@ isassigned(c) = UTF8PROC_CATEGORY_CN < category_code(c) <= UTF8PROC_CATEGORY_CO
 """
  islowercase(c::AbstractChar) -> Bool
 
-Tests whether a character is a lowercase letter.
-A character is classified as lowercase if it belongs to Unicode category Ll,
-Letter: Lowercase.
+Tests whether a character is a lowercase letter (according to the Unicode
+standard's `Lowercase` derived property).
 
 See also: [`isuppercase`](@ref).
 
@@ -298,16 +297,15 @@ julia> islowercase('❤')
 false
 ```
 """
-islowercase(c::AbstractChar) = category_code(c) == UTF8PROC_CATEGORY_LL
+islowercase(c::AbstractChar) = ismalformed(c) ? false : Bool(ccall(:utf8proc_islower, Cint, (UInt32,), UInt32(c)))
 
 # true for Unicode upper and mixed case
 
 """
  isuppercase(c::AbstractChar) -> Bool
 
-Tests whether a character is an uppercase letter.
-A character is classified as uppercase if it belongs to Unicode category Lu,
-Letter: Uppercase, or Lt, Letter: Titlecase.
+Tests whether a character is an uppercase letter (according to the Unicode
+standard's `Uppercase` derived property).
 
 See also: [`islowercase`](@ref).
 
@@ -323,10 +321,7 @@ julia> isuppercase('❤')
 false
 ```
 """
-function isuppercase(c::AbstractChar)
- cat = category_code(c)
- cat == UTF8PROC_CATEGORY_LU || cat == UTF8PROC_CATEGORY_LT
-end
+isuppercase(c::AbstractChar) = ismalformed(c) ? false : Bool(ccall(:utf8proc_isupper, Cint, (UInt32,), UInt32(c)))
 
 """
  iscased(c::AbstractChar) -> Bool

stdlib/Unicode/test/runtests.jl

@@ -93,25 +93,28 @@ end
 @testset "#5939 uft8proc character predicates" begin
  alower=['a', 'd', 'j', 'y', 'z']
  ulower=['α', 'β', 'γ', 'δ', 'ф', 'я']
- for c in vcat(alower,ulower)
+ for c in vcat(alower,ulower,['ª'])
  @test islowercase(c) == true
  @test isuppercase(c) == false
  @test isdigit(c) == false
  @test isnumeric(c) == false
  end
 
  aupper=['A', 'D', 'J', 'Y', 'Z']
- uupper= ['Δ', 'Γ', 'Π', 'Ψ', 'Dž', 'Ж', 'Д']
+ uupper= ['Δ', 'Γ', 'Π', 'Ψ', 'Ж', 'Д']
 
- for c in vcat(aupper,uupper)
+ for c in vcat(aupper,uupper,['Ⓐ'])
  @test islowercase(c) == false
  @test isuppercase(c) == true
  @test isdigit(c) == false
  @test isnumeric(c) == false
  end
 
+ @test !isuppercase('Dž') # titlecase is not uppercase
+ @test Base.Unicode.iscased('Dž') # but is "cased"
+
  nocase=['א','ﺵ']
- alphas=vcat(alower,ulower,aupper,uupper,nocase)
+ alphas=vcat(alower,ulower,aupper,uupper,nocase,['Dž'])
 
  for c in alphas
  @test isletter(c) == true