001/*
002 * Copyright 2017 Product Mog LLC.
003 *
004 * Licensed under the Apache License, Version 2.0 (the "License");
005 * you may not use this file except in compliance with the License.
006 * You may obtain a copy of the License at
007 *
008 * http://www.apache.org/licenses/LICENSE-2.0
009 *
010 * Unless required by applicable law or agreed to in writing, software
011 * distributed under the License is distributed on an "AS IS" BASIS,
012 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
013 * See the License for the specific language governing permissions and
014 * limitations under the License.
015 */
016
017package com.lokalized;
018
019import com.lokalized.LocalizedString.LanguageFormTranslation;
020import com.lokalized.LocalizedString.LanguageFormTranslationRange;
021import com.lokalized.MinimalJson.Json;
022import com.lokalized.MinimalJson.JsonArray;
023import com.lokalized.MinimalJson.JsonObject;
024import com.lokalized.MinimalJson.JsonObject.Member;
025import com.lokalized.MinimalJson.JsonValue;
026
027import javax.annotation.Nonnull;
028import javax.annotation.concurrent.ThreadSafe;
029import java.io.File;
030import java.io.IOException;
031import java.net.URL;
032import java.nio.file.Files;
033import java.nio.file.Path;
034import java.util.ArrayList;
035import java.util.Arrays;
036import java.util.Collections;
037import java.util.HashSet;
038import java.util.LinkedHashMap;
039import java.util.LinkedHashSet;
040import java.util.List;
041import java.util.Locale;
042import java.util.Map;
043import java.util.Set;
044import java.util.TreeMap;
045import java.util.logging.Logger;
046import java.util.stream.Collectors;
047
048import static java.lang.String.format;
049import static java.nio.charset.StandardCharsets.UTF_8;
050import static java.util.Objects.requireNonNull;
051
052/**
053 * Utility methods for loading localized strings files.
054 *
055 * @author <a href="https://revetkn.com">Mark Allen</a>
056 */
057@ThreadSafe
058public final class LocalizedStringLoader {
059  @Nonnull
060  private static final Set<String> SUPPORTED_LANGUAGE_TAGS;
061  @Nonnull
062  private static final Map<String, LanguageForm> SUPPORTED_LANGUAGE_FORMS_BY_NAME;
063  @Nonnull
064  private static final Logger LOGGER;
065
066  static {
067    LOGGER = Logger.getLogger(LoggerType.LOCALIZED_STRING_LOADER.getLoggerName());
068
069    SUPPORTED_LANGUAGE_TAGS = Collections.unmodifiableSet(Arrays.stream(Locale.getAvailableLocales())
070        .map(locale -> locale.toLanguageTag())
071        .collect(Collectors.toSet()));
072
073    Set<LanguageForm> supportedLanguageForms = new LinkedHashSet<>();
074    supportedLanguageForms.addAll(Arrays.asList(Gender.values()));
075    supportedLanguageForms.addAll(Arrays.asList(Cardinality.values()));
076    supportedLanguageForms.addAll(Arrays.asList(Ordinality.values()));
077
078    Map<String, LanguageForm> supportedLanguageFormsByName = new LinkedHashMap<>();
079
080    for (LanguageForm languageForm : supportedLanguageForms) {
081      if (!languageForm.getClass().isEnum())
082        throw new IllegalArgumentException(format("The %s interface must be implemented by enum types. %s is not an enum",
083            LanguageForm.class.getSimpleName(), languageForm.getClass().getSimpleName()));
084
085      String languageFormName = ((Enum<?>) languageForm).name();
086      LanguageForm existingLanguageForm = supportedLanguageFormsByName.get(languageFormName);
087
088      if (existingLanguageForm != null)
089        throw new IllegalArgumentException(format("There is already a language form %s.%s whose name collides with %s.%s. " +
090                "Language form names must be unique", existingLanguageForm.getClass().getSimpleName(), languageFormName,
091            languageForm.getClass().getSimpleName(), languageFormName));
092
093      // Massage Cardinality to match file format, e.g. "ONE" -> "CARDINALITY_ONE"
094      if (languageForm instanceof Cardinality)
095        languageFormName = LocalizedStringUtils.localizedStringNameForCardinalityName(languageFormName);
096
097      // Massage Ordinality to match file format, e.g. "ONE" -> "ORDINALITY_ONE"
098      if (languageForm instanceof Ordinality)
099        languageFormName = LocalizedStringUtils.localizedStringNameForOrdinalityName(languageFormName);
100
101      supportedLanguageFormsByName.put(languageFormName, languageForm);
102    }
103
104    SUPPORTED_LANGUAGE_FORMS_BY_NAME = Collections.unmodifiableMap(supportedLanguageFormsByName);
105  }
106
107  private LocalizedStringLoader() {
108    // Non-instantiable
109  }
110
111  /**
112   * Loads all localized string files present in the specified package on the classpath.
113   * <p>
114   * Filenames must correspond to the IETF BCP 47 language tag format.
115   * <p>
116   * Example filenames:
117   * <ul>
118   * <li>{@code en}</li>
119   * <li>{@code es-MX}</li>
120   * <li>{@code nan-Hant-TW}</li>
121   * </ul>
122   * <p>
123   * Like any classpath reference, packages are separated using the {@code /} character.
124   * <p>
125   * Example package names:
126   * <ul>
127   * <li>{@code strings}
128   * <li>{@code com/lokalized/strings}
129   * </ul>
130   * <p>
131   * Note: this implementation only scans the specified package, it does not descend into child packages.
132   *
133   * @param classpathPackage location of a package on the classpath, not null
134   * @return per-locale sets of localized strings, not null
135   * @throws LocalizedStringLoadingException if an error occurs while loading localized string files
136   */
137  @Nonnull
138  public static Map<Locale, Set<LocalizedString>> loadFromClasspath(@Nonnull String classpathPackage) {
139    requireNonNull(classpathPackage);
140
141    ClassLoader classLoader = LocalizedStringLoader.class.getClassLoader();
142    URL url = classLoader.getResource(classpathPackage);
143
144    if (url == null)
145      throw new LocalizedStringLoadingException(format("Unable to find package '%s' on the classpath", classpathPackage));
146
147    return loadFromDirectory(new File(url.getFile()));
148  }
149
150  /**
151   * Loads all localized string files present in the specified directory.
152   * <p>
153   * Filenames must correspond to the IETF BCP 47 language tag format.
154   * <p>
155   * Example filenames:
156   * <ul>
157   * <li>{@code en}</li>
158   * <li>{@code es-MX}</li>
159   * <li>{@code nan-Hant-TW}</li>
160   * </ul>
161   * <p>
162   * Note: this implementation only scans the specified directory, it does not descend into child directories.
163   *
164   * @param directory directory in which to search for localized string files, not null
165   * @return per-locale sets of localized strings, not null
166   * @throws LocalizedStringLoadingException if an error occurs while loading localized string files
167   */
168  @Nonnull
169  public static Map<Locale, Set<LocalizedString>> loadFromFilesystem(@Nonnull Path directory) {
170    requireNonNull(directory);
171    return loadFromDirectory(directory.toFile());
172  }
173
174  // TODO: should we expose methods for loading a single file?
175
176  /**
177   * Loads all localized string files present in the specified directory.
178   *
179   * @param directory directory in which to search for localized string files, not null
180   * @return per-locale sets of localized strings, not null
181   * @throws LocalizedStringLoadingException if an error occurs while loading localized string files
182   */
183  @Nonnull
184  private static Map<Locale, Set<LocalizedString>> loadFromDirectory(@Nonnull File directory) {
185    requireNonNull(directory);
186
187    if (!directory.exists())
188      throw new LocalizedStringLoadingException(format("Location '%s' does not exist",
189          directory));
190
191    if (!directory.isDirectory())
192      throw new LocalizedStringLoadingException(format("Location '%s' exists but is not a directory",
193          directory));
194
195    Map<Locale, Set<LocalizedString>> localizedStringsByLocale =
196        new TreeMap<>((locale1, locale2) -> locale1.toLanguageTag().compareTo(locale2.toLanguageTag()));
197
198    File[] files = directory.listFiles();
199
200    if (files != null) {
201      for (File file : files) {
202        String languageTag = file.getName();
203
204        if (SUPPORTED_LANGUAGE_TAGS.contains(languageTag)) {
205          LOGGER.fine(format("Loading localized strings file '%s'...", languageTag));
206          Locale locale = Locale.forLanguageTag(file.getName());
207          localizedStringsByLocale.put(locale, parseLocalizedStringsFile(file));
208        } else {
209          LOGGER.fine(format("File '%s' does not correspond to a known language tag, skipping...", languageTag));
210        }
211      }
212    }
213
214    return Collections.unmodifiableMap(localizedStringsByLocale);
215  }
216
217  /**
218   * Parses out a set of localized strings from the given file.
219   *
220   * @param file the file to parse, not null
221   * @return the set of localized strings contained in the file, not null
222   * @throws LocalizedStringLoadingException if an error occurs while parsing the localized string file
223   */
224  @Nonnull
225  private static Set<LocalizedString> parseLocalizedStringsFile(@Nonnull File file) {
226    requireNonNull(file);
227
228    String canonicalPath;
229
230    try {
231      canonicalPath = file.getCanonicalPath();
232    } catch (IOException e) {
233      throw new LocalizedStringLoadingException(
234          format("Unable to determine canonical path for localized strings file %s", file), e);
235    }
236
237    if (!Files.isRegularFile(file.toPath()))
238      throw new LocalizedStringLoadingException(format("%s is not a regular file", canonicalPath));
239
240    String localizedStringsFileContents;
241
242    try {
243      localizedStringsFileContents = new String(Files.readAllBytes(file.toPath()), UTF_8).trim();
244    } catch (IOException e) {
245      throw new LocalizedStringLoadingException(format("Unable to load localized strings file contents for %s",
246          canonicalPath), e);
247    }
248
249    if ("".equals(localizedStringsFileContents))
250      return Collections.emptySet();
251
252    Set<LocalizedString> localizedStrings = new HashSet<>();
253    JsonValue outerJsonValue = Json.parse(localizedStringsFileContents);
254
255    if (!outerJsonValue.isObject())
256      throw new LocalizedStringLoadingException(format("%s: a localized strings file must be comprised of a single JSON object", canonicalPath));
257
258    JsonObject outerJsonObject = outerJsonValue.asObject();
259
260    for (Member member : outerJsonObject) {
261      String key = member.getName();
262      JsonValue value = member.getValue();
263      localizedStrings.add(parseLocalizedString(canonicalPath, key, value));
264    }
265
266    return Collections.unmodifiableSet(localizedStrings);
267  }
268
269  /**
270   * Parses "toplevel" localized string data.
271   * <p>
272   * Operates recursively if alternatives are encountered.
273   *
274   * @param canonicalPath the unique path to the file (or URL) being parsed, used for error reporting. not null
275   * @param key           the toplevel translation key, not null
276   * @param jsonValue     the toplevel translation value - might be a simple string, might be a complex object. not null
277   * @return a localized string instance, not null
278   * @throws LocalizedStringLoadingException if an error occurs while parsing the localized string file
279   */
280  @Nonnull
281  private static LocalizedString parseLocalizedString(@Nonnull String canonicalPath, @Nonnull String key, @Nonnull JsonValue jsonValue) {
282    requireNonNull(canonicalPath);
283    requireNonNull(key);
284    requireNonNull(jsonValue);
285
286    if (jsonValue.isString()) {
287      // Simple case - just a key and a value, no translation rules
288      //
289      // Example format:
290      //
291      // {
292      //   "Hello, world!" : "Приветствую, мир"
293      // }
294
295      String translation = jsonValue.asString();
296
297      if (translation == null)
298        throw new LocalizedStringLoadingException(format("%s: a translation is required for key '%s'", canonicalPath, key));
299
300      return new LocalizedString.Builder(key).translation(translation).build();
301    } else if (jsonValue.isObject()) {
302      // More complex case, there can be placeholders and alternatives.
303      //
304      // Example format:
305      //
306      // {
307      //   "I read {{bookCount}} books" : {
308      //     "translation" : "I read {{bookCount}} {{books}}",
309      //     "commentary" : "Message shown when user achieves her book-reading goal for the month",
310      //     "placeholders" : {
311      //       "books" : {
312      //         "value" : "bookCount",
313      //         "translations" : {
314      //           "ONE" : "book",
315      //           "OTHER" : "books"
316      //         }
317      //       }
318      //     },
319      //     "alternatives" : [
320      //       {
321      //         "bookCount == 0" : {
322      //           "translation" : "I haven't read any books"
323      //         }
324      //       }
325      //     ]
326      //   }
327      // }
328
329      JsonObject localizedStringObject = jsonValue.asObject();
330
331      String translation = null;
332
333      JsonValue translationJsonValue = localizedStringObject.get("translation");
334
335      if (translationJsonValue != null && !translationJsonValue.isNull()) {
336        if (!translationJsonValue.isString())
337          throw new LocalizedStringLoadingException(format("%s: translation must be a string for key '%s'", canonicalPath, key));
338
339        translation = translationJsonValue.asString();
340      }
341
342      String commentary = null;
343
344      JsonValue commentaryJsonValue = localizedStringObject.get("commentary");
345
346      if (commentaryJsonValue != null && !commentaryJsonValue.isNull()) {
347        if (!commentaryJsonValue.isString())
348          throw new LocalizedStringLoadingException(format("%s: commentary must be a string for key '%s'", canonicalPath, key));
349
350        commentary = commentaryJsonValue.asString();
351      }
352
353      Map<String, LanguageFormTranslation> languageFormTranslationsByPlaceholder = new LinkedHashMap<>();
354
355      JsonValue placeholdersJsonValue = localizedStringObject.get("placeholders");
356
357      if (placeholdersJsonValue != null && !placeholdersJsonValue.isNull()) {
358        if (!placeholdersJsonValue.isObject())
359          throw new LocalizedStringLoadingException(format("%s: the placeholders value must be an object. Key is '%s'", canonicalPath, key));
360
361        JsonObject placeholdersJsonObject = placeholdersJsonValue.asObject();
362
363        for (Member placeholderMember : placeholdersJsonObject) {
364          String placeholderKey = placeholderMember.getName();
365          JsonValue placeholderJsonValue = placeholderMember.getValue();
366          String value = null;
367          LanguageFormTranslationRange rangeValue = null;
368
369          if (!placeholderJsonValue.isObject())
370            throw new LocalizedStringLoadingException(format("%s: the placeholder value must be an object. Key is '%s'", canonicalPath, key));
371
372          JsonObject placeholderJsonObject = placeholderJsonValue.asObject();
373
374          JsonValue valueJsonValue = placeholderJsonObject.get("value");
375          JsonValue rangeJsonValue = placeholderJsonObject.get("range");
376          boolean hasValue = valueJsonValue != null && !valueJsonValue.isNull();
377          boolean hasRangeValue = rangeJsonValue != null && !rangeJsonValue.isNull();
378
379          if (!hasValue && !hasRangeValue)
380            throw new LocalizedStringLoadingException(format("%s: a placeholder translation value or range is required. Key is '%s'", canonicalPath, key));
381
382          if (hasValue && hasRangeValue)
383            throw new LocalizedStringLoadingException(format("%s: a placeholder translation cannot have both a value and a range. Key is '%s'", canonicalPath, key));
384
385          if (hasRangeValue) {
386            if (!rangeJsonValue.isObject())
387              throw new LocalizedStringLoadingException(format("%s: the placeholder translation range must be an object. Key is '%s'", canonicalPath, key));
388
389            JsonObject rangeJsonObject = rangeJsonValue.asObject();
390            JsonValue rangeValueStartJsonValue = rangeJsonObject.get("start");
391            JsonValue rangeValueEndJsonValue = rangeJsonObject.get("end");
392
393            if (rangeValueStartJsonValue == null || rangeValueStartJsonValue.isNull())
394              throw new LocalizedStringLoadingException(format("%s: a placeholder translation range start is required. Key is '%s'", canonicalPath, key));
395
396            if (rangeValueEndJsonValue == null || rangeValueEndJsonValue.isNull())
397              throw new LocalizedStringLoadingException(format("%s: a placeholder translation range end is required. Key is '%s'", canonicalPath, key));
398
399            if (!rangeValueStartJsonValue.isString())
400              throw new LocalizedStringLoadingException(format("%s: a placeholder translation range start must be a string. Key is '%s'", canonicalPath, key));
401
402            if (!rangeValueEndJsonValue.isString())
403              throw new LocalizedStringLoadingException(format("%s: a placeholder translation range end must be a string. Key is '%s'", canonicalPath, key));
404
405            rangeValue = new LanguageFormTranslationRange(rangeValueStartJsonValue.asString(), rangeValueEndJsonValue.asString());
406          } else {
407            if (!valueJsonValue.isString())
408              throw new LocalizedStringLoadingException(format("%s: a placeholder translation value must be a string. Key is '%s'", canonicalPath, key));
409
410            value = valueJsonValue.asString();
411          }
412
413          JsonValue translationsJsonValue = placeholderJsonObject.get("translations");
414
415          if (translationsJsonValue == null || translationsJsonValue.isNull())
416            continue;
417
418          if (!translationsJsonValue.isObject())
419            throw new LocalizedStringLoadingException(format("%s: the placeholder translations value must be an object. Key is '%s'", canonicalPath, key));
420
421          Map<LanguageForm, String> translationsByLanguageForm = new LinkedHashMap<>();
422
423          JsonObject translationsJsonObject = translationsJsonValue.asObject();
424
425          for (Member translationMember : translationsJsonObject) {
426            String languageFormTranslationKey = translationMember.getName();
427            JsonValue languageFormTranslationJsonValue = translationMember.getValue();
428            LanguageForm languageForm = SUPPORTED_LANGUAGE_FORMS_BY_NAME.get(languageFormTranslationKey);
429
430            if (languageForm == null)
431              throw new LocalizedStringLoadingException(format("%s: unexpected placeholder translation language form encountered. Key is '%s'. " +
432                      "You provided '%s', valid values are [%s]", canonicalPath, key, languageFormTranslationKey,
433                  SUPPORTED_LANGUAGE_FORMS_BY_NAME.keySet().stream().collect(Collectors.joining(", "))));
434
435            if (!languageFormTranslationJsonValue.isString())
436              throw new LocalizedStringLoadingException(format("%s: the placeholder translation value must be a string. Key is '%s'", canonicalPath, key));
437
438            translationsByLanguageForm.put(languageForm, languageFormTranslationJsonValue.asString());
439          }
440
441          LanguageFormTranslation languageFormTranslation = rangeValue != null
442              ? new LanguageFormTranslation(rangeValue, translationsByLanguageForm)
443              : new LanguageFormTranslation(value, translationsByLanguageForm);
444
445          languageFormTranslationsByPlaceholder.put(placeholderKey, languageFormTranslation);
446        }
447      }
448
449      List<LocalizedString> alternatives = new ArrayList<>();
450
451      JsonValue alternativesJsonValue = localizedStringObject.get("alternatives");
452
453      if (alternativesJsonValue != null && !alternativesJsonValue.isNull()) {
454        if (!alternativesJsonValue.isArray())
455          throw new LocalizedStringLoadingException(format("%s: alternatives must be an array. Key is '%s'", canonicalPath, key));
456
457        JsonArray alternativesJsonArray = alternativesJsonValue.asArray();
458
459        for (JsonValue alternativeJsonValue : alternativesJsonArray) {
460          if (alternativeJsonValue == null || alternativeJsonValue.isNull())
461            continue;
462
463          JsonObject outerJsonObject = alternativeJsonValue.asObject();
464
465          if (!outerJsonObject.isObject())
466            throw new LocalizedStringLoadingException(format("%s: alternative value must be an object. Key is '%s'", canonicalPath, key));
467
468          for (Member member : outerJsonObject) {
469            String alternativeKey = member.getName();
470            JsonValue alternativeValue = member.getValue();
471            alternatives.add(parseLocalizedString(canonicalPath, alternativeKey, alternativeValue));
472          }
473        }
474      }
475
476      return new LocalizedString.Builder(key)
477          .translation(translation)
478          .commentary(commentary)
479          .languageFormTranslationsByPlaceholder(languageFormTranslationsByPlaceholder)
480          .alternatives(alternatives)
481          .build();
482    } else {
483      throw new LocalizedStringLoadingException(format("%s: either a translation string or object value is required for key '%s'",
484          canonicalPath, key));
485    }
486  }
487}