LibUnicode: Parse and generate the Unicode locale list patterns dataset

This data informs consumers how to join lists of values. For example,
in en-US, the list ["a", "b", "c"] formatted to a string should become
"a, b, and c".

Meta/Lagom/Tools/CodeGenerators/LibUnicode/GenerateUnicodeLocale.cpp

@@ -21,6 +21,15 @@
 #include <LibCore/File.h>
 #include <LibUnicode/Locale.h>
 
+struct ListPatterns {
+ String type;
+ String style;
+ String start;
+ String middle;
+ String end;
+ String pair;
+};
+
 struct Locale {
  String language;
  Optional<String> territory;
@@ -29,6 +38,7 @@ struct Locale {
  HashMap<String, String> territories;
  HashMap<String, String> scripts;
  HashMap<String, String> currencies;
+ Vector<ListPatterns> list_patterns;
 };
 
 struct CanonicalLanguageID {
@@ -50,6 +60,8 @@ struct UnicodeLocaleData {
  Vector<String> scripts;
  Vector<String> variants;
  Vector<String> currencies;
+ Vector<String> list_pattern_types;
+ Vector<String> list_pattern_styles;
  HashMap<String, String> language_aliases;
  HashMap<String, String> territory_aliases;
  HashMap<String, String> script_aliases;
@@ -293,6 +305,58 @@ static void parse_locale_scripts(String locale_path, UnicodeLocaleData& locale_d
  });
 }
 
+static void parse_locale_list_patters(String misc_path, UnicodeLocaleData& locale_data, Locale& locale)
+{
+ LexicalPath list_patterns_path(move(misc_path));
+ list_patterns_path = list_patterns_path.append("listPatterns.json"sv);
+ VERIFY(Core::File::exists(list_patterns_path.string()));
+
+ auto list_patterns_file_or_error = Core::File::open(list_patterns_path.string(), Core::OpenMode::ReadOnly);
+ VERIFY(!list_patterns_file_or_error.is_error());
+
+ auto list_patterns = JsonParser(list_patterns_file_or_error.value()->read_all()).parse();
+ VERIFY(list_patterns.has_value());
+
+ auto const& main_object = list_patterns->as_object().get("main"sv);
+ auto const& locale_object = main_object.as_object().get(list_patterns_path.parent().basename());
+ auto const& list_patterns_object = locale_object.as_object().get("listPatterns"sv);
+
+ auto list_pattern_type = [](StringView key) {
+ if (key.contains("type-standard"sv))
+ return "conjunction"sv;
+ if (key.contains("type-or"sv))
+ return "disjunction"sv;
+ if (key.contains("type-unit"sv))
+ return "unit"sv;
+ VERIFY_NOT_REACHED();
+ };
+
+ auto list_pattern_style = [](StringView key) {
+ if (key.contains("short"sv))
+ return "short"sv;
+ if (key.contains("narrow"sv))
+ return "narrow"sv;
+ return "long"sv;
+ };
+
+ list_patterns_object.as_object().for_each_member([&](auto const& key, JsonValue const& value) {
+ auto type = list_pattern_type(key);
+ auto style = list_pattern_style(key);
+
+ auto start = value.as_object().get("start"sv).as_string();
+ auto middle = value.as_object().get("middle"sv).as_string();
+ auto end = value.as_object().get("end"sv).as_string();
+ auto pair = value.as_object().get("2"sv).as_string();
+
+ if (!locale_data.list_pattern_types.contains_slow(type))
+ locale_data.list_pattern_types.append(type);
+ if (!locale_data.list_pattern_styles.contains_slow(style))
+ locale_data.list_pattern_styles.append(style);
+
+ locale.list_patterns.append({ move(type), move(style), move(start), move(middle), move(end), move(pair) });
+ });
+}
+
 static void parse_locale_currencies(String numbers_path, UnicodeLocaleData& locale_data, Locale& locale)
 {
  LexicalPath currencies_path(move(numbers_path));
@@ -333,9 +397,10 @@ static Core::DirIterator path_to_dir_iterator(String path)
  return iterator;
 }
 
-static void parse_all_locales(String core_path, String locale_names_path, String numbers_path, UnicodeLocaleData& locale_data)
+static void parse_all_locales(String core_path, String locale_names_path, String misc_path, String numbers_path, UnicodeLocaleData& locale_data)
 {
  auto locale_names_iterator = path_to_dir_iterator(move(locale_names_path));
+ auto misc_iterator = path_to_dir_iterator(move(misc_path));
  auto numbers_iterator = path_to_dir_iterator(move(numbers_path));
 
  LexicalPath core_supplemental_path(move(core_path));
@@ -356,6 +421,14 @@ static void parse_all_locales(String core_path, String locale_names_path, String
  parse_locale_scripts(locale_path, locale_data, locale);
  }
 
+ while (misc_iterator.has_next()) {
+ auto misc_path = misc_iterator.next_full_path();
+ VERIFY(Core::File::is_directory(misc_path));
+
+ auto& locale = locale_data.locales.ensure(LexicalPath::basename(misc_path));
+ parse_locale_list_patters(misc_path, locale_data, locale);
+ }
+
  while (numbers_iterator.has_next()) {
  auto numbers_path = numbers_iterator.next_full_path();
  VERIFY(Core::File::is_directory(numbers_path));
@@ -423,6 +496,8 @@ namespace Unicode {
  generate_enum("ScriptTag"sv, {}, locale_data.scripts);
  generate_enum("Currency"sv, {}, locale_data.currencies);
  generate_enum("Variant"sv, {}, locale_data.variants);
+ generate_enum("ListPatternType"sv, {}, locale_data.list_pattern_types);
+ generate_enum("ListPatternStyle"sv, {}, locale_data.list_pattern_styles);
 
  generator.append(R"~~~(
 namespace Detail {
@@ -444,6 +519,10 @@ Optional<StringView> resolve_script_tag_alias(StringView const& script_tag);
 Optional<StringView> get_locale_currency_mapping(StringView locale, StringView currency);
 Optional<Currency> currency_from_string(StringView const& currency);
 
+Optional<ListPatterns> get_locale_list_pattern_mapping(StringView locale, StringView list_pattern_type, StringView list_pattern_style);
+Optional<ListPatternType> list_pattern_type_from_string(StringView const& list_pattern_type);
+Optional<ListPatternStyle> list_pattern_style_from_string(StringView const& list_pattern_style);
+
 Optional<StringView> resolve_variant_alias(StringView const& variant);
 Optional<StringView> resolve_subdivision_alias(StringView const& subdivision);
 
@@ -476,6 +555,15 @@ static void generate_unicode_locale_implementation(Core::File& file, UnicodeLoca
 #include <LibUnicode/UnicodeLocale.h>
 
 namespace Unicode {
+
+struct Patterns {
+ ListPatternType type;
+ ListPatternStyle style;
+ StringView start;
+ StringView middle;
+ StringView end;
+ StringView pair;
+};
 )~~~");
 
  auto format_mapping_name = [](StringView format, StringView name) {
@@ -507,7 +595,7 @@ namespace Unicode {
  generator.append(String::formatted(" }}, {}", list.size()));
  };
 
- auto append_mapping_list = [&](String name, auto const& keys, auto const& mappings) {
+ auto append_string_list = [&](String name, auto const& keys, auto const& mappings) {
  generator.set("name", name);
  generator.set("size", String::number(keys.size()));
 
@@ -539,21 +627,46 @@ static constexpr Array<StringView, @size@> @name@ { {
 )~~~");
  };
 
- auto append_mapping = [&](StringView name, StringView format, auto const& keys, auto get_mapping_callback) {
+ auto append_list_patterns = [&](StringView name, Vector<ListPatterns> const& list_patterns) {
+ generator.set("name", name);
+ generator.set("size", String::number(list_patterns.size()));
+
+ generator.append(R"~~~(
+static constexpr Array<Patterns, @size@> @name@ { {)~~~");
+
+ for (auto const& list_pattern : list_patterns) {
+ generator.set("type"sv, String::formatted("ListPatternType::{}", format_identifier({}, list_pattern.type)));
+ generator.set("style"sv, String::formatted("ListPatternStyle::{}", format_identifier({}, list_pattern.style)));
+ generator.set("start"sv, String::formatted("\"{}\"sv", list_pattern.start));
+ generator.set("middle"sv, String::formatted("\"{}\"sv", list_pattern.middle));
+ generator.set("end"sv, String::formatted("\"{}\"sv", list_pattern.end));
+ generator.set("pair"sv, String::formatted("\"{}\"sv", list_pattern.pair));
+
+ generator.append(R"~~~(
+ { @type@, @style@, @start@, @middle@, @end@, @pair@ },)~~~");
+ }
+
+ generator.append(R"~~~(
+} };
+)~~~");
+ };
+
+ auto append_mapping = [&](StringView type, StringView name, StringView format, auto format_list_callback) {
  Vector<String> mapping_names;
 
  for (auto const& locale : locale_data.locales) {
  auto mapping_name = format_mapping_name(format, locale.key);
- append_mapping_list(mapping_name, keys, get_mapping_callback(locale.value));
+ format_list_callback(mapping_name, locale.value);
  mapping_names.append(move(mapping_name));
  }
 
  quick_sort(mapping_names);
 
+ generator.set("type", type);
  generator.set("name", name);
  generator.set("size", String::number(locale_data.locales.size()));
  generator.append(R"~~~(
-static constexpr Array<Span<StringView const>, @size@> @name@ { {
+static constexpr Array<Span<@type@ const>, @size@> @name@ { {
  )~~~");
 
  constexpr size_t max_values_per_row = 10;
@@ -577,10 +690,11 @@ static constexpr Array<Span<StringView const>, @size@> @name@ { {
 )~~~");
  };
 
- append_mapping("s_languages"sv, "s_languages_{}", locale_data.languages, [](auto const& value) { return value.languages; });
- append_mapping("s_territories"sv, "s_territories_{}", locale_data.territories, [](auto const& value) { return value.territories; });
- append_mapping("s_scripts"sv, "s_scripts_{}", locale_data.scripts, [](auto const& value) { return value.scripts; });
- append_mapping("s_currencies"sv, "s_currencies_{}", locale_data.currencies, [](auto const& value) { return value.currencies; });
+ append_mapping("StringView"sv, "s_languages"sv, "s_languages_{}", [&](auto const& name, auto const& value) { append_string_list(name, locale_data.languages, value.languages); });
+ append_mapping("StringView"sv, "s_territories"sv, "s_territories_{}", [&](auto const& name, auto const& value) { append_string_list(name, locale_data.territories, value.territories); });
+ append_mapping("StringView"sv, "s_scripts"sv, "s_scripts_{}", [&](auto const& name, auto const& value) { append_string_list(name, locale_data.scripts, value.scripts); });
+ append_mapping("StringView"sv, "s_currencies"sv, "s_currencies_{}", [&](auto const& name, auto const& value) { append_string_list(name, locale_data.currencies, value.currencies); });
+ append_mapping("Patterns"sv, "s_list_patterns"sv, "s_list_patterns_{}", [&](auto const& name, auto const& value) { append_list_patterns(name, value.list_patterns); });
 
  generator.append(R"~~~(
 struct CanonicalLanguageID {
@@ -866,7 +980,35 @@ Optional<StringView> resolve_@enum_snake@_alias(StringView const& @enum_snake@)
  append_alias_search("variant"sv, locale_data.variant_aliases);
  append_alias_search("subdivision"sv, locale_data.subdivision_aliases);
 
+ append_from_string("ListPatternType"sv, "list_pattern_type"sv, locale_data.list_pattern_types);
+ append_from_string("ListPatternStyle"sv, "list_pattern_style"sv, locale_data.list_pattern_styles);
+
  generator.append(R"~~~(
+Optional<ListPatterns> get_locale_list_pattern_mapping(StringView locale, StringView list_pattern_type, StringView list_pattern_style)
+{
+ auto locale_value = locale_from_string(locale);
+ if (!locale_value.has_value())
+ return {};
+
+ auto type_value = list_pattern_type_from_string(list_pattern_type);
+ if (!type_value.has_value())
+ return {};
+
+ auto style_value = list_pattern_style_from_string(list_pattern_style);
+ if (!style_value.has_value())
+ return {};
+
+ auto locale_index = to_underlying(*locale_value) - 1; // Subtract 1 because 0 == Locale::None.
+ auto const& locale_list_patterns = s_list_patterns.at(locale_index);
+
+ for (auto const& list_patterns : locale_list_patterns) {
+ if ((list_patterns.type == type_value) && (list_patterns.style == style_value))
+ return ListPatterns { list_patterns.start, list_patterns.middle, list_patterns.end, list_patterns.pair };
+ }
+
+ return {};
+}
+
 void resolve_complex_language_aliases(Unicode::LanguageID& language_id)
 {
  for (auto const& map : s_complex_alias) {
@@ -969,7 +1111,7 @@ int main(int argc, char** argv)
  auto generated_implementation_file = open_file(generated_implementation_path, "-c/--generated-implementation-path", Core::OpenMode::ReadWrite);
 
  UnicodeLocaleData locale_data;
- parse_all_locales(core_path, locale_names_path, numbers_path, locale_data);
+ parse_all_locales(core_path, locale_names_path, misc_path, numbers_path, locale_data);
 
  generate_unicode_locale_header(generated_header_file, locale_data);
  generate_unicode_locale_implementation(generated_implementation_file, locale_data);

Userland/Libraries/LibUnicode/Forward.h

@@ -13,6 +13,8 @@ namespace Unicode {
 enum class Condition : u8;
 enum class GeneralCategory : u8;
 enum class Language : u8;
+enum class ListPatternStyle : u8;
+enum class ListPatternType : u8;
 enum class Locale : u16;
 enum class Property : u8;
 enum class Script : u8;
@@ -21,6 +23,7 @@ enum class WordBreakProperty : u8;
 
 struct Keyword;
 struct LanguageID;
+struct ListPatterns;
 struct LocaleExtension;
 struct LocaleID;
 struct OtherExtension;

Userland/Libraries/LibUnicode/Locale.cpp

@@ -798,6 +798,15 @@ Optional<StringView> get_locale_currency_mapping([[maybe_unused]] StringView loc
 #endif
 }
 
+Optional<ListPatterns> get_locale_list_patterns([[maybe_unused]] StringView locale, [[maybe_unused]] StringView type, [[maybe_unused]] StringView style)
+{
+#if ENABLE_UNICODE_DATA
+ return Detail::get_locale_list_pattern_mapping(locale, type, style);
+#else
+ return {};
+#endif
+}
+
 Optional<StringView> resolve_language_alias(StringView language)
 {
 #if ENABLE_UNICODE_DATA

Userland/Libraries/LibUnicode/Locale.h

@@ -78,6 +78,13 @@ struct LocaleID {
  Vector<String> private_use_extensions {};
 };
 
+struct ListPatterns {
+ StringView start;
+ StringView middle;
+ StringView end;
+ StringView pair;
+};
+
 // Note: These methods only verify that the provided strings match the EBNF grammar of the
 // Unicode identifier subtag (i.e. no validation is done that the tags actually exist).
 constexpr bool is_unicode_language_subtag(StringView subtag)
@@ -130,6 +137,7 @@ Optional<StringView> get_locale_language_mapping(StringView locale, StringView l
 Optional<StringView> get_locale_territory_mapping(StringView locale, StringView territory);
 Optional<StringView> get_locale_script_mapping(StringView locale, StringView script);
 Optional<StringView> get_locale_currency_mapping(StringView locale, StringView currency);
+Optional<ListPatterns> get_locale_list_patterns(StringView locale, StringView type, StringView style);
 
 Optional<StringView> resolve_language_alias(StringView language);
 Optional<StringView> resolve_territory_alias(StringView territory);