001/* 002 * Copyright 2017 Product Mog LLC. 003 * 004 * Licensed under the Apache License, Version 2.0 (the "License"); 005 * you may not use this file except in compliance with the License. 006 * You may obtain a copy of the License at 007 * 008 * http://www.apache.org/licenses/LICENSE-2.0 009 * 010 * Unless required by applicable law or agreed to in writing, software 011 * distributed under the License is distributed on an "AS IS" BASIS, 012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 013 * See the License for the specific language governing permissions and 014 * limitations under the License. 015 */ 016 017package com.lokalized; 018 019import com.lokalized.LocalizedString.LanguageFormTranslation; 020import com.lokalized.LocalizedString.LanguageFormTranslationRange; 021import com.lokalized.MinimalJson.Json; 022import com.lokalized.MinimalJson.JsonArray; 023import com.lokalized.MinimalJson.JsonObject; 024import com.lokalized.MinimalJson.JsonObject.Member; 025import com.lokalized.MinimalJson.JsonValue; 026 027import javax.annotation.Nonnull; 028import javax.annotation.concurrent.ThreadSafe; 029import java.io.File; 030import java.io.IOException; 031import java.net.URL; 032import java.nio.file.Files; 033import java.nio.file.Path; 034import java.util.ArrayList; 035import java.util.Arrays; 036import java.util.Collections; 037import java.util.HashSet; 038import java.util.LinkedHashMap; 039import java.util.LinkedHashSet; 040import java.util.List; 041import java.util.Locale; 042import java.util.Map; 043import java.util.Set; 044import java.util.TreeMap; 045import java.util.logging.Logger; 046import java.util.stream.Collectors; 047 048import static java.lang.String.format; 049import static java.nio.charset.StandardCharsets.UTF_8; 050import static java.util.Objects.requireNonNull; 051 052/** 053 * Utility methods for loading localized strings files. 054 * 055 * @author <a href="https://revetkn.com">Mark Allen</a> 056 */ 057@ThreadSafe 058public final class LocalizedStringLoader { 059 @Nonnull 060 private static final Set<String> SUPPORTED_LANGUAGE_TAGS; 061 @Nonnull 062 private static final Map<String, LanguageForm> SUPPORTED_LANGUAGE_FORMS_BY_NAME; 063 @Nonnull 064 private static final Logger LOGGER; 065 066 static { 067 LOGGER = Logger.getLogger(LoggerType.LOCALIZED_STRING_LOADER.getLoggerName()); 068 069 SUPPORTED_LANGUAGE_TAGS = Collections.unmodifiableSet(Arrays.stream(Locale.getAvailableLocales()) 070 .map(locale -> locale.toLanguageTag()) 071 .collect(Collectors.toSet())); 072 073 Set<LanguageForm> supportedLanguageForms = new LinkedHashSet<>(); 074 supportedLanguageForms.addAll(Arrays.asList(Gender.values())); 075 supportedLanguageForms.addAll(Arrays.asList(Cardinality.values())); 076 supportedLanguageForms.addAll(Arrays.asList(Ordinality.values())); 077 078 Map<String, LanguageForm> supportedLanguageFormsByName = new LinkedHashMap<>(); 079 080 for (LanguageForm languageForm : supportedLanguageForms) { 081 if (!languageForm.getClass().isEnum()) 082 throw new IllegalArgumentException(format("The %s interface must be implemented by enum types. %s is not an enum", 083 LanguageForm.class.getSimpleName(), languageForm.getClass().getSimpleName())); 084 085 String languageFormName = ((Enum<?>) languageForm).name(); 086 LanguageForm existingLanguageForm = supportedLanguageFormsByName.get(languageFormName); 087 088 if (existingLanguageForm != null) 089 throw new IllegalArgumentException(format("There is already a language form %s.%s whose name collides with %s.%s. " + 090 "Language form names must be unique", existingLanguageForm.getClass().getSimpleName(), languageFormName, 091 languageForm.getClass().getSimpleName(), languageFormName)); 092 093 // Massage Cardinality to match file format, e.g. "ONE" -> "CARDINALITY_ONE" 094 if (languageForm instanceof Cardinality) 095 languageFormName = LocalizedStringUtils.localizedStringNameForCardinalityName(languageFormName); 096 097 // Massage Ordinality to match file format, e.g. "ONE" -> "ORDINALITY_ONE" 098 if (languageForm instanceof Ordinality) 099 languageFormName = LocalizedStringUtils.localizedStringNameForOrdinalityName(languageFormName); 100 101 supportedLanguageFormsByName.put(languageFormName, languageForm); 102 } 103 104 SUPPORTED_LANGUAGE_FORMS_BY_NAME = Collections.unmodifiableMap(supportedLanguageFormsByName); 105 } 106 107 private LocalizedStringLoader() { 108 // Non-instantiable 109 } 110 111 /** 112 * Loads all localized string files present in the specified package on the classpath. 113 * <p> 114 * Filenames must correspond to the IETF BCP 47 language tag format. 115 * <p> 116 * Example filenames: 117 * <ul> 118 * <li>{@code en}</li> 119 * <li>{@code es-MX}</li> 120 * <li>{@code nan-Hant-TW}</li> 121 * </ul> 122 * <p> 123 * Like any classpath reference, packages are separated using the {@code /} character. 124 * <p> 125 * Example package names: 126 * <ul> 127 * <li>{@code strings} 128 * <li>{@code com/lokalized/strings} 129 * </ul> 130 * <p> 131 * Note: this implementation only scans the specified package, it does not descend into child packages. 132 * 133 * @param classpathPackage location of a package on the classpath, not null 134 * @return per-locale sets of localized strings, not null 135 * @throws LocalizedStringLoadingException if an error occurs while loading localized string files 136 */ 137 @Nonnull 138 public static Map<Locale, Set<LocalizedString>> loadFromClasspath(@Nonnull String classpathPackage) { 139 requireNonNull(classpathPackage); 140 141 ClassLoader classLoader = LocalizedStringLoader.class.getClassLoader(); 142 URL url = classLoader.getResource(classpathPackage); 143 144 if (url == null) 145 throw new LocalizedStringLoadingException(format("Unable to find package '%s' on the classpath", classpathPackage)); 146 147 return loadFromDirectory(new File(url.getFile())); 148 } 149 150 /** 151 * Loads all localized string files present in the specified directory. 152 * <p> 153 * Filenames must correspond to the IETF BCP 47 language tag format. 154 * <p> 155 * Example filenames: 156 * <ul> 157 * <li>{@code en}</li> 158 * <li>{@code es-MX}</li> 159 * <li>{@code nan-Hant-TW}</li> 160 * </ul> 161 * <p> 162 * Note: this implementation only scans the specified directory, it does not descend into child directories. 163 * 164 * @param directory directory in which to search for localized string files, not null 165 * @return per-locale sets of localized strings, not null 166 * @throws LocalizedStringLoadingException if an error occurs while loading localized string files 167 */ 168 @Nonnull 169 public static Map<Locale, Set<LocalizedString>> loadFromFilesystem(@Nonnull Path directory) { 170 requireNonNull(directory); 171 return loadFromDirectory(directory.toFile()); 172 } 173 174 // TODO: should we expose methods for loading a single file? 175 176 /** 177 * Loads all localized string files present in the specified directory. 178 * 179 * @param directory directory in which to search for localized string files, not null 180 * @return per-locale sets of localized strings, not null 181 * @throws LocalizedStringLoadingException if an error occurs while loading localized string files 182 */ 183 @Nonnull 184 private static Map<Locale, Set<LocalizedString>> loadFromDirectory(@Nonnull File directory) { 185 requireNonNull(directory); 186 187 if (!directory.exists()) 188 throw new LocalizedStringLoadingException(format("Location '%s' does not exist", 189 directory)); 190 191 if (!directory.isDirectory()) 192 throw new LocalizedStringLoadingException(format("Location '%s' exists but is not a directory", 193 directory)); 194 195 Map<Locale, Set<LocalizedString>> localizedStringsByLocale = 196 new TreeMap<>((locale1, locale2) -> locale1.toLanguageTag().compareTo(locale2.toLanguageTag())); 197 198 File[] files = directory.listFiles(); 199 200 if (files != null) { 201 for (File file : files) { 202 String languageTag = file.getName(); 203 204 if (SUPPORTED_LANGUAGE_TAGS.contains(languageTag)) { 205 LOGGER.fine(format("Loading localized strings file '%s'...", languageTag)); 206 Locale locale = Locale.forLanguageTag(file.getName()); 207 localizedStringsByLocale.put(locale, parseLocalizedStringsFile(file)); 208 } else { 209 LOGGER.fine(format("File '%s' does not correspond to a known language tag, skipping...", languageTag)); 210 } 211 } 212 } 213 214 return Collections.unmodifiableMap(localizedStringsByLocale); 215 } 216 217 /** 218 * Parses out a set of localized strings from the given file. 219 * 220 * @param file the file to parse, not null 221 * @return the set of localized strings contained in the file, not null 222 * @throws LocalizedStringLoadingException if an error occurs while parsing the localized string file 223 */ 224 @Nonnull 225 private static Set<LocalizedString> parseLocalizedStringsFile(@Nonnull File file) { 226 requireNonNull(file); 227 228 String canonicalPath; 229 230 try { 231 canonicalPath = file.getCanonicalPath(); 232 } catch (IOException e) { 233 throw new LocalizedStringLoadingException( 234 format("Unable to determine canonical path for localized strings file %s", file), e); 235 } 236 237 if (!Files.isRegularFile(file.toPath())) 238 throw new LocalizedStringLoadingException(format("%s is not a regular file", canonicalPath)); 239 240 String localizedStringsFileContents; 241 242 try { 243 localizedStringsFileContents = new String(Files.readAllBytes(file.toPath()), UTF_8).trim(); 244 } catch (IOException e) { 245 throw new LocalizedStringLoadingException(format("Unable to load localized strings file contents for %s", 246 canonicalPath), e); 247 } 248 249 if ("".equals(localizedStringsFileContents)) 250 return Collections.emptySet(); 251 252 Set<LocalizedString> localizedStrings = new HashSet<>(); 253 JsonValue outerJsonValue = Json.parse(localizedStringsFileContents); 254 255 if (!outerJsonValue.isObject()) 256 throw new LocalizedStringLoadingException(format("%s: a localized strings file must be comprised of a single JSON object", canonicalPath)); 257 258 JsonObject outerJsonObject = outerJsonValue.asObject(); 259 260 for (Member member : outerJsonObject) { 261 String key = member.getName(); 262 JsonValue value = member.getValue(); 263 localizedStrings.add(parseLocalizedString(canonicalPath, key, value)); 264 } 265 266 return Collections.unmodifiableSet(localizedStrings); 267 } 268 269 /** 270 * Parses "toplevel" localized string data. 271 * <p> 272 * Operates recursively if alternatives are encountered. 273 * 274 * @param canonicalPath the unique path to the file (or URL) being parsed, used for error reporting. not null 275 * @param key the toplevel translation key, not null 276 * @param jsonValue the toplevel translation value - might be a simple string, might be a complex object. not null 277 * @return a localized string instance, not null 278 * @throws LocalizedStringLoadingException if an error occurs while parsing the localized string file 279 */ 280 @Nonnull 281 private static LocalizedString parseLocalizedString(@Nonnull String canonicalPath, @Nonnull String key, @Nonnull JsonValue jsonValue) { 282 requireNonNull(canonicalPath); 283 requireNonNull(key); 284 requireNonNull(jsonValue); 285 286 if (jsonValue.isString()) { 287 // Simple case - just a key and a value, no translation rules 288 // 289 // Example format: 290 // 291 // { 292 // "Hello, world!" : "Приветствую, мир" 293 // } 294 295 String translation = jsonValue.asString(); 296 297 if (translation == null) 298 throw new LocalizedStringLoadingException(format("%s: a translation is required for key '%s'", canonicalPath, key)); 299 300 return new LocalizedString.Builder(key).translation(translation).build(); 301 } else if (jsonValue.isObject()) { 302 // More complex case, there can be placeholders and alternatives. 303 // 304 // Example format: 305 // 306 // { 307 // "I read {{bookCount}} books" : { 308 // "translation" : "I read {{bookCount}} {{books}}", 309 // "commentary" : "Message shown when user achieves her book-reading goal for the month", 310 // "placeholders" : { 311 // "books" : { 312 // "value" : "bookCount", 313 // "translations" : { 314 // "ONE" : "book", 315 // "OTHER" : "books" 316 // } 317 // } 318 // }, 319 // "alternatives" : [ 320 // { 321 // "bookCount == 0" : { 322 // "translation" : "I haven't read any books" 323 // } 324 // } 325 // ] 326 // } 327 // } 328 329 JsonObject localizedStringObject = jsonValue.asObject(); 330 331 String translation = null; 332 333 JsonValue translationJsonValue = localizedStringObject.get("translation"); 334 335 if (translationJsonValue != null && !translationJsonValue.isNull()) { 336 if (!translationJsonValue.isString()) 337 throw new LocalizedStringLoadingException(format("%s: translation must be a string for key '%s'", canonicalPath, key)); 338 339 translation = translationJsonValue.asString(); 340 } 341 342 String commentary = null; 343 344 JsonValue commentaryJsonValue = localizedStringObject.get("commentary"); 345 346 if (commentaryJsonValue != null && !commentaryJsonValue.isNull()) { 347 if (!commentaryJsonValue.isString()) 348 throw new LocalizedStringLoadingException(format("%s: commentary must be a string for key '%s'", canonicalPath, key)); 349 350 commentary = commentaryJsonValue.asString(); 351 } 352 353 Map<String, LanguageFormTranslation> languageFormTranslationsByPlaceholder = new LinkedHashMap<>(); 354 355 JsonValue placeholdersJsonValue = localizedStringObject.get("placeholders"); 356 357 if (placeholdersJsonValue != null && !placeholdersJsonValue.isNull()) { 358 if (!placeholdersJsonValue.isObject()) 359 throw new LocalizedStringLoadingException(format("%s: the placeholders value must be an object. Key is '%s'", canonicalPath, key)); 360 361 JsonObject placeholdersJsonObject = placeholdersJsonValue.asObject(); 362 363 for (Member placeholderMember : placeholdersJsonObject) { 364 String placeholderKey = placeholderMember.getName(); 365 JsonValue placeholderJsonValue = placeholderMember.getValue(); 366 String value = null; 367 LanguageFormTranslationRange rangeValue = null; 368 369 if (!placeholderJsonValue.isObject()) 370 throw new LocalizedStringLoadingException(format("%s: the placeholder value must be an object. Key is '%s'", canonicalPath, key)); 371 372 JsonObject placeholderJsonObject = placeholderJsonValue.asObject(); 373 374 JsonValue valueJsonValue = placeholderJsonObject.get("value"); 375 JsonValue rangeJsonValue = placeholderJsonObject.get("range"); 376 boolean hasValue = valueJsonValue != null && !valueJsonValue.isNull(); 377 boolean hasRangeValue = rangeJsonValue != null && !rangeJsonValue.isNull(); 378 379 if (!hasValue && !hasRangeValue) 380 throw new LocalizedStringLoadingException(format("%s: a placeholder translation value or range is required. Key is '%s'", canonicalPath, key)); 381 382 if (hasValue && hasRangeValue) 383 throw new LocalizedStringLoadingException(format("%s: a placeholder translation cannot have both a value and a range. Key is '%s'", canonicalPath, key)); 384 385 if (hasRangeValue) { 386 if (!rangeJsonValue.isObject()) 387 throw new LocalizedStringLoadingException(format("%s: the placeholder translation range must be an object. Key is '%s'", canonicalPath, key)); 388 389 JsonObject rangeJsonObject = rangeJsonValue.asObject(); 390 JsonValue rangeValueStartJsonValue = rangeJsonObject.get("start"); 391 JsonValue rangeValueEndJsonValue = rangeJsonObject.get("end"); 392 393 if (rangeValueStartJsonValue == null || rangeValueStartJsonValue.isNull()) 394 throw new LocalizedStringLoadingException(format("%s: a placeholder translation range start is required. Key is '%s'", canonicalPath, key)); 395 396 if (rangeValueEndJsonValue == null || rangeValueEndJsonValue.isNull()) 397 throw new LocalizedStringLoadingException(format("%s: a placeholder translation range end is required. Key is '%s'", canonicalPath, key)); 398 399 if (!rangeValueStartJsonValue.isString()) 400 throw new LocalizedStringLoadingException(format("%s: a placeholder translation range start must be a string. Key is '%s'", canonicalPath, key)); 401 402 if (!rangeValueEndJsonValue.isString()) 403 throw new LocalizedStringLoadingException(format("%s: a placeholder translation range end must be a string. Key is '%s'", canonicalPath, key)); 404 405 rangeValue = new LanguageFormTranslationRange(rangeValueStartJsonValue.asString(), rangeValueEndJsonValue.asString()); 406 } else { 407 if (!valueJsonValue.isString()) 408 throw new LocalizedStringLoadingException(format("%s: a placeholder translation value must be a string. Key is '%s'", canonicalPath, key)); 409 410 value = valueJsonValue.asString(); 411 } 412 413 JsonValue translationsJsonValue = placeholderJsonObject.get("translations"); 414 415 if (translationsJsonValue == null || translationsJsonValue.isNull()) 416 continue; 417 418 if (!translationsJsonValue.isObject()) 419 throw new LocalizedStringLoadingException(format("%s: the placeholder translations value must be an object. Key is '%s'", canonicalPath, key)); 420 421 Map<LanguageForm, String> translationsByLanguageForm = new LinkedHashMap<>(); 422 423 JsonObject translationsJsonObject = translationsJsonValue.asObject(); 424 425 for (Member translationMember : translationsJsonObject) { 426 String languageFormTranslationKey = translationMember.getName(); 427 JsonValue languageFormTranslationJsonValue = translationMember.getValue(); 428 LanguageForm languageForm = SUPPORTED_LANGUAGE_FORMS_BY_NAME.get(languageFormTranslationKey); 429 430 if (languageForm == null) 431 throw new LocalizedStringLoadingException(format("%s: unexpected placeholder translation language form encountered. Key is '%s'. " + 432 "You provided '%s', valid values are [%s]", canonicalPath, key, languageFormTranslationKey, 433 SUPPORTED_LANGUAGE_FORMS_BY_NAME.keySet().stream().collect(Collectors.joining(", ")))); 434 435 if (!languageFormTranslationJsonValue.isString()) 436 throw new LocalizedStringLoadingException(format("%s: the placeholder translation value must be a string. Key is '%s'", canonicalPath, key)); 437 438 translationsByLanguageForm.put(languageForm, languageFormTranslationJsonValue.asString()); 439 } 440 441 LanguageFormTranslation languageFormTranslation = rangeValue != null 442 ? new LanguageFormTranslation(rangeValue, translationsByLanguageForm) 443 : new LanguageFormTranslation(value, translationsByLanguageForm); 444 445 languageFormTranslationsByPlaceholder.put(placeholderKey, languageFormTranslation); 446 } 447 } 448 449 List<LocalizedString> alternatives = new ArrayList<>(); 450 451 JsonValue alternativesJsonValue = localizedStringObject.get("alternatives"); 452 453 if (alternativesJsonValue != null && !alternativesJsonValue.isNull()) { 454 if (!alternativesJsonValue.isArray()) 455 throw new LocalizedStringLoadingException(format("%s: alternatives must be an array. Key is '%s'", canonicalPath, key)); 456 457 JsonArray alternativesJsonArray = alternativesJsonValue.asArray(); 458 459 for (JsonValue alternativeJsonValue : alternativesJsonArray) { 460 if (alternativeJsonValue == null || alternativeJsonValue.isNull()) 461 continue; 462 463 JsonObject outerJsonObject = alternativeJsonValue.asObject(); 464 465 if (!outerJsonObject.isObject()) 466 throw new LocalizedStringLoadingException(format("%s: alternative value must be an object. Key is '%s'", canonicalPath, key)); 467 468 for (Member member : outerJsonObject) { 469 String alternativeKey = member.getName(); 470 JsonValue alternativeValue = member.getValue(); 471 alternatives.add(parseLocalizedString(canonicalPath, alternativeKey, alternativeValue)); 472 } 473 } 474 } 475 476 return new LocalizedString.Builder(key) 477 .translation(translation) 478 .commentary(commentary) 479 .languageFormTranslationsByPlaceholder(languageFormTranslationsByPlaceholder) 480 .alternatives(alternatives) 481 .build(); 482 } else { 483 throw new LocalizedStringLoadingException(format("%s: either a translation string or object value is required for key '%s'", 484 canonicalPath, key)); 485 } 486 } 487}