Support Bulk Download from GlotPress

lib/fastlane/plugin/wpmreleasetoolkit/helper/android/android_strings_file_writer.rb

@@ -0,0 +1,29 @@
+require 'fastlane_core/ui/ui'
+require 'fileutils'
+
+module Fastlane
+  module Helper
+    module Android
+      module StringsFileWriter
+        # @param [String] dir path to destination directory
+        # @param [Locale] locale the locale to write the file for
+        # @param [File, IO] io The File IO containing the translations downloaded from GlotPress
+        def self.write_app_translations_file(dir:, locale:, io:)
+          # `dir` is typically `src/main/res/` here
+          return unless Locale.valid?(locale, :android)
+
+          dest = File.join(dir, locale.android_path)
+          FileUtils.mkdir_p(File.dirname(dest))
+
+          # TODO: reorder XML nodes alphabetically, for easier diffs
+          #   xml = Nokogiri::XML(io, nil, Encoding::UTF_8.to_s)
+          #   # … reorder nodes …
+          #   File.open(main, 'w:UTF-8') { |f| f.write(xml.to_xml(indent: 4)) }
+          # FIXME: For now, just copy blindly until we get time to implement node reordering
+          UI.message("Writing: #{dest}")
+          IO.copy_stream(io, dest)
+        end
+      end
+    end
+  end
+end

lib/fastlane/plugin/wpmreleasetoolkit/helper/fastlane_metadata_file_writer.rb

@@ -0,0 +1,102 @@
+require 'fastlane_core/ui/ui'
+require 'fileutils'
+
+module Fastlane
+  module Helper
+    module FastlaneMetadataFilesWriter
+
+      # A model/struct defining a rule on how to process and map metadata from GlotPress into txt files
+      #
+      # @param [String] key The key in the GlotPress export for the metadata
+      # @param [Int] max_len The maximum length allowed by the App Store / Play Store for that key.
+      #        Note: If the translation for `key` exceeds the specified `max_len`, we will try to find an alternate key named `#{key}_short` by convention.
+      # @param [String] filename The (relative) path to the `.txt` file to write that metadata to
+      #
+      MetadataRule = Struct.new(:key, :max_len, :filename) do
+        # The common standardized set of Metadata rules for an Android project
+        def self.android_rules(version_name:, version_code:)
+          suffix = version_name.gsub('.', '')
+          [
+            MetadataRule.new("release_note_#{suffix}", 500, File.join('changelogs', "#{version_code}.txt")),
+            MetadataRule.new('play_store_app_title', 30, 'title.txt'),
+            MetadataRule.new('play_store_promo', 80, 'short_description.txt'),
+            MetadataRule.new('play_store_desc', 4000, 'full_description.txt'),
+          ]
+        end
+
+        # The common standardized set of Metadata rules for an Android project
+        def self.ios_rules(version_name:)
+          suffix = version_name.gsub('.', '')
+          [
+            MetadataRule.new("release_note_#{suffix}", 4000, 'release_notes.txt'),
+            MetadataRule.new('app_store_name', 30, 'name.txt'),
+            MetadataRule.new('app_store_subtitle', 30, 'subtitle.txt'),
+            MetadataRule.new('app_store_description', 4000, 'description.txt'),
+            MetadataRule.new('app_store_keywords', 100, 'keywords.txt'),
+          ]
+        end
+      end
+
+      # Visit each key/value pair of a translations Hash, and yield keys and matching translations from it based on the passed `MetadataRules`,
+      # trying any potential fallback key if the translation exceeds the max limit, and yielding each found and valid entry to the caller.
+      #
+      # @param [#read] io
+      # @param [Array<MetadataRule>] rules List of rules for each key
+      # @param [Block] rule_for_unknown_key An optional block called when a key that does not match any of the rules is encountered.
+      #        The block will receive a [String] (key) and must return a `MetadataRule` instance (or nil)
+      #
+      # @yield [String, MetadataRule, String] yield each (key, matching_rule, value) tuple found in the JSON, after resolving alternates for values exceeding max length
+      #        Note that if both translations for the key and its (optional) shorter alternate exceeds the max_len, it will still `yield` but with a `nil` value
+      #
+      def self.visit(translations:, rules:, rule_for_unknown_key:)
+        translations.each do |key, value|
+          next if key.nil? || key.end_with?('_short') # skip if alternate key
+
+          rule = rules.find { |r| r.key == key }
+          rule = rule_for_unknown_key.call(key) if rule.nil? && !rule_for_unknown_key.nil?
+          next if rule.nil?
+
+          if rule.max_len != nil && value.length > rule.max_len
+            UI.warning "Translation for #{key} is too long (#{value.length}), trying shorter alternate #{key}."
+            short_key = "#{key}_short"
+            value = json[short_key]
+            if value.nil?
+              UI.warning "No shorter alternate (#{short_key}) available, skipping entirely."
+              yield key, rule, nil
+              next
+            end
+            if value.length > rule.max_len
+              UI.warning "Translation alternate for #{short_key} was too long too (#{value.length}), skipping entirely."
+              yield short_key, rule, nil
+              next
+            end
+          end
+          yield key, rule, value
+        end
+      end
+
+      # Write the `.txt` files to disk for the given exported translation file (typically a JSON export) based on the `MetadataRules` provided
+      #
+      # @param [String] locale_dir the path to the locale directory (e.g. `fastlane/metadata/android/fr`) to write the `.txt` files to
+      # @param [Hash<String,String>] translations The hash of translations (key => translation) to visit based on `MetadataRules` then write to disk.
+      # @param [Array<MetadaataRule>] rules The list of fixed `MetadataRule` to use to extract the expected metadata from the `translations`
+      # @param [Block] rule_for_unknown_key An optional block called when a key that does not match any of the rules is encountered.
+      #        The block will receive a [String] (key) and must return a `MetadataRule` instance (or nil)
+      #
+      def self.write(locale_dir:, translations:, rules:, &rule_for_unknown_key)
+        self.visit(translations: translations, rules: rules, rule_for_unknown_key: rule_for_unknown_key) do |_key, rule, value|
+          dest = File.join(locale_dir, rule.filename)
+          if value.nil? && File.exist?(dest)
+            # Key found in JSON was rejected for being too long. Delete file
+            UI.verbose("Deleting file #{dest}")
+            FileUtils.rm(dest)
+          elsif value
+            UI.verbose("Writing file #{dest}")
+            FileUtils.mkdir_p(File.dirname(dest))
+            File.write(dest, value.chomp)
+          end
+        end
+      end
+    end
+  end
+end

lib/fastlane/plugin/wpmreleasetoolkit/helper/glotpress_downloader.rb

@@ -0,0 +1,93 @@
+require 'fastlane_core/ui/ui'
+require 'json'
+require 'open-uri'
+require 'zip'
+
+module Fastlane
+  module Helper
+    class GPDownloader
+      REQUEST_HEADERS = { 'User-Agent' => Wpmreleasetoolkit::USER_AGENT }
+
+      module FORMAT
+        ANDROID = 'android'
+        IOS = 'strings'
+        JSON = 'json'
+      end
+
+      # The host of the GlotPress instance. e.g. `'translate.wordpress.org'`
+      attr_accessor :host
+      # The path of the project in GlotPress. e.g. `'apps/ios/release-notes'`
+      attr_accessor :project
+
+      def initialize(host:, project:)
+        @host = host
+        @project = project
+      end
+
+      # @param [String] gp_locale
+      # @param [String] format Typically `'android'`, `'strings'` or `'json'`
+      # @param [Hash<String,String>] filters
+      #
+      # @yield [IO] the corresponding downloaded IO content
+      #
+      # @note For this case, `project_url` is on the form 'https://translate.wordpress.org/projects/apps/ios/release-notes'
+      def download_locale(gp_locale:, format:, filters: { status: 'current'})
+        query_params = filters.transform_keys { |k| "filters[#{k}]" }.merge(format: format)
+        uri = URI::HTTPS.build(host: host, path: File.join('/', 'projects', project, gp_locale, 'default', 'export-translations'), query: URI.encode_www_form(query_params))
+
+        UI.message "Downloading #{uri}"
+        io = begin
+          uri.open(REQUEST_HEADERS)
+        rescue StandardError => e
+          UI.error "Error downloading #{gp_locale} - #{e.message}"
+          return
+        end
+        UI.message "Download done."
+        yield io
+      end
+
+      # @param [String] format Typically `'android'`, `'strings'` or `'json'`
+      # @param [Hash<String,String>] filters
+      #
+      # @yield For each locale, a tuple of [String], [IO] corresponding to the glotpress locale code and IO content
+      #
+      # @note requires the GlotPress instance to have the Bulk Downloader plugin installed
+      # @note For this case, `project_url` is on the form 'https://translate.wordpress.org/exporter/apps/android/dev/'
+      def download_all_locales(format:, filters: { status: 'current'})
+        query_params = filters.transform_keys { |k| "filters[#{k}]" }.merge('export-format': format)
+        uri = URI::HTTPS.build(host: host, path: File.join('/', 'exporter', project, '-do'), query: URI.encode_www_form(query_params))
+        UI.message "Downloading #{uri}"
+        zip_stream = uri.open(REQUEST_HEADERS)
+        UI.message "Download done."
+
+        Zip::File.open_buffer(zip_stream) do |zip_file|
+          zip_file.each do |entry|
+            next if entry.name.end_with?('/') && entry.size.zero?
+
+            prefix = File.dirname(entry.name).gsub(/[0-9-]*$/, '') + '-'
+            locale = File.basename(entry.name, File.extname(entry.name)).delete_prefix(prefix)
+            UI.message "- Found locale in ZIP: #{locale}"
+
+            yield locale, entry.get_input_stream
+          end
+        end
+      end
+
+      # Takes a GlotPress JSON export and transform it to a simple `Hash` of key => value pairs
+      #
+      # Since the JSON format for GlotPress exports is a bit odd, with JSON keys actually being a concatenation of actual
+      # copy key and source copy, and values being an array, this allows us to convert this odd export format to a more
+      # usable structure.
+      #
+      # @param [#read] io The `File` or `IO` to read the JSON data exported from GlotPress
+      def parse_json_export(io:)
+        json = JSON.parse(io.read)
+        json.map do |composite_key, values|
+          key = composite_key.split(/\u0004/).first # composite_key is a concatenation of key + \u0004 + source]
+          value = values.first # Each value in the JSON Hash is an Array of all the translations; but if we provided the right filter, the first one should always be the right one
+          [key, value]
+        end.to_h
+      end
+    end # class
+  end # module
+end # module

lib/fastlane/plugin/wpmreleasetoolkit/helper/ios/ios_strings_file_writer.rb

@@ -0,0 +1,24 @@
+require 'fastlane_core/ui/ui'
+require 'fileutils'
+
+module Fastlane
+  module Helper
+    module Ios
+      module StringsFileWriter
+        # @param [String] dir path to destination directory
+        # @param [Locale] locale the locale to write the file for
+        # @param [File, IO] io The File IO containing the translations downloaded from GlotPress
+        def self.write_app_translations_file(dir:, locale:, io:)
+          # `dir` is typically `WordPress/Resources/` here
+          return unless Locale.valid?(locale, :ios)
+
+          dest = File.join(dir, locale.ios_path)
+          FileUtils.mkdir_p(File.dirname(dest))
+          UI.message("Writing: #{dest}")
+          IO.copy_stream(io, dest)
+        end
+      end
+    end
+  end
+end
+

lib/fastlane/plugin/wpmreleasetoolkit/models/locale.rb

@@ -0,0 +1,29 @@
+require 'fastlane_core/ui/ui'
+
+module Fastlane
+  module Wpmreleasetoolkit
+    Locale = Struct.new(:glotpress, :android, :playstore, :ios, :appstore, keyword_init: true) do
+      def android_path
+        File.join("values-#{self.android}", 'strings.xml')
+      end
+
+      def ios_path
+        File.join("#{self.ios}.lproj", 'Localizable.strings')
+      end
+
+      def self.valid?(locale, *keys)
+        if locale.nil?
+          UI.warning("Locale is unknown")
+          return false
+        end
+        keys.each do |key|
+          if locale[key].nil?
+            UI.warning("Locale #{locale} is missing required key #{key}")
+            return false
+          end
+        end
+        return true
+      end
+    end
+  end
+end

lib/fastlane/plugin/wpmreleasetoolkit/models/locales.rb

@@ -0,0 +1,152 @@
+module Fastlane
+  module Wpmreleasetoolkit
+    class Locales
+      ALL_KNOWN_LOCALES = [
+        Locale.new(glotpress: 'ar',    android: 'ar',     playstore: 'ar'),
+        Locale.new(glotpress: 'de',    android: 'de',     playstore: 'de-DE'),
+        Locale.new(glotpress: 'en-gb', android: 'en-rGB', playstore: 'en-US'),
+        Locale.new(glotpress: 'es',    android: 'es',     playstore: 'es-ES'),
+        Locale.new(glotpress: 'fr-ca', android: 'fr-rCA', playstore: 'fr-CA'),
+        Locale.new(glotpress: 'fr',    android: 'fr',     playstore: 'fr-FR', ios: 'fr-FR', appstore: 'fr-FR'),
+        Locale.new(glotpress: 'he',    android: 'he',     playstore: 'iw-IL'),
+        Locale.new(glotpress: 'id',    android: 'id',     playstore: 'id'),
+        Locale.new(glotpress: 'it',    android: 'it',     playstore: 'it-IT'),
+        Locale.new(glotpress: 'ja',    android: 'ja',     playstore: 'ja-JP'),
+        Locale.new(glotpress: 'ko',    android: 'ko',     playstore: 'ko-KR'),
+        Locale.new(glotpress: 'nl',    android: 'nl',     playstore: 'nl-NL'),
+        Locale.new(glotpress: 'pl',    android: 'pl',     playstore: 'pl-PL'),
+        Locale.new(glotpress: 'pt-br', android: 'pt-rBR', playstore: 'pt-BR', ios: 'pt-BR', appstore: 'pt-BR'),
+        Locale.new(glotpress: 'ru',    android: 'ru',     playstore: 'ru-RU'),
+        Locale.new(glotpress: 'sr',    android: 'sr',     playstore: 'sr'),
+        Locale.new(glotpress: 'sv',    android: 'sv',     playstore: 'sv-SE'),
+        Locale.new(glotpress: 'th',    android: 'th',     playstore: 'th'),
+        Locale.new(glotpress: 'tr',    android: 'tr',     playstore: 'tr-TR'),
+        Locale.new(glotpress: 'vi',    android: 'vi',     playstore: 'vi'),
+        Locale.new(glotpress: 'zh-cn', android: 'zh-rCN', playstore: 'zh-CN', ios: 'zh-Hans', appstore: 'zh-Hans'),
+        Locale.new(glotpress: 'zh-tw', android: 'zh-rTW', playstore: 'zh-TW', ios: 'zh-Hant', appstore: 'zh-Hant'),
+        Locale.new(glotpress: 'az',    android: 'az'),
+        Locale.new(glotpress: 'el',    android: 'el')
+        # FIXME: Complete the list with ios/app_store properties for all, and extending to more locales
+      ]
+
+      MAG16_GP_CODES = %w[ar de es fr he id it ja ko nl pt-br ru sv tr zh-cn zh-tw].freeze
+
+      ##############
+
+      # [Array<Locale>]
+      attr_accessor :locales
+
+      # @param [Array<Locale>,Array<Hash>] locales
+      def initialize(locales = ALL_KNOWN_LOCALES)
+        @locales = locales.map { |l| l.is_a?(Locale) ? l : Locale.new(l) }
+      end
+
+      ##############
+      # @!group Filter `Locales` based on locale codes
+
+      # Return the list of locales matching the gp_codes passed as input parameters
+      #
+      # @param [String...] codes The locale codes to get the Locales for
+      # @param [Symbol] key_name The name of the `Locale` property to use to filter those locales by.
+      #        Defaults to `:glotpress` (= the `codes` param is expected to be _GlotPress_ locale codes by default)
+      # @return [Locales]
+      def self.[](*codes, key_name: :glotpress)
+        locales = ALL_KNOWN_LOCALES.select { |l| codes.include?(l[key_name.to_sym]) }
+        Locales.new(locales)
+      end
+
+      # Find a single given locale amongst the set of all known locales
+      #
+      # @param [String] code
+      # @param [Symbol] key_name The name of the `Locale` property to use to filter those locales by.
+      #        Defaults to `:glotpress` (= the `codes` param is expected to be _GlotPress_ locale codes by default)
+      # @return [Locale?] The known locale matching the provided code, or `nil` if no known locale was found.
+      def self.find(code, key_name: :glotpress)
+        Locales.all.find(code, key_name: key_name)
+      end
+
+      # Find a single given locale amongst the set of locales registered in this `Locales` instance
+      #
+      # @param [String] code
+      # @param [Symbol] key_name The name of the `Locale` property to use to filter those locales by.
+      #        Defaults to `:glotpress` (= the `codes` param is expected to be _GlotPress_ locale codes by default)
+      # @return [Locale?] The known locale matching the provided code, or `nil` if no known locale was found.
+      def find(code, key_name: :glotpress)
+        @locales.find { |l| code == l[key_name.to_sym] }
+      end
+
+      # @!endgroup
+      ##############
+
+      ##############
+      # @!group Common locale sets
+
+      def self.all
+        Locales.new(ALL_KNOWN_LOCALES)
+      end
+
+      def self.mag16
+        Locales[*MAG16_GP_CODES]
+      end
+
+      # @!endgroup
+      ##############
+
+      ##############
+      # @!group Locales set arithmetics
+
+      # Substraction
+      def -(other)
+        Locales.new(self.locales - other.locales)
+      end
+
+      # Intersection
+      def &(other)
+        Locales.new(self.locales & other.locales)
+      end
+
+      # Addition (without deduplication guarantee)
+      def +(other)
+        Locales.new(self.locales + other.locales)
+      end
+
+      # Union (with deduplication)
+      def |(other)
+        Locales.new(self.locales | other.locales)
+      end
+
+      # @!endgroup
+      ##############
+
+      ##############
+      # @!group Conversion to other types and iteration
+
+      def each
+        @locales.each { |l| yield l }
+      end
+
+      # Constructs a `Hash` whose keys are the locale code for `key_sym` (e.g. `:glotpress`) and corresponding values are the locale code for `value_sym` (e.g. `:android`)
+      # Example: `Locales.mag16.to_hash(:glotpress, :android)`
+      def to_hash(key_sym, value_sym)
+        Hash.new(
+          @locales.map { |l| [l[key_sym], l[value_sym]] }
+        )
+      end
+
+      def to_a
+        if block_given?
+          @locales.map { |l| yield l }
+        else
+          @locales
+        end
+      end
+
+      def to_s
+        "\#<Locales: [\n  #{@locales.join("\n  ")}\n]>"
+      end
+
+      # @!endgroup
+      ##############
+    end
+  end
+end

spec/glotpress_downloader_spec.rb

@@ -0,0 +1,98 @@
+# NOTE: This is not really a spec but a demo script instead that I used to test my implementation.
+# FIXME: Convert this to an actual spec with unit test cases and stubs/fixtures
+
+module GlotpressDownloaderDemo
+  DOTORG = 'translate.wordpress.org'
+  DOTCOM = 'translate.wordpress.com'
+  WP_ANDROID = { host: DOTORG, project: 'apps/android/dev' }
+  WP_IOS = { host: DOTORG, project: 'apps/ios/dev' }
+  WC_ANDROID = { host: DOTCOM, project: 'woocommerce/woocommerce-android' }
+  WC_IOS = { host: DOTCOM, project: 'woocommerce/woocommerce-ios' }
+
+  EXPORT_FMT = Fastlane::Helper::GPDownloader::FORMAT
+  FastlaneMetadataFilesWriter = Fastlane::Helper::FastlaneMetadataFilesWriter
+
+  EXAMPLE_OUTPUT_DIR = 'MyTestApp'
+
+  # Example Usages for App Translation
+
+  def demo_android_app_translations_bulk
+    output_dir = File.join(EXAMPLE_OUTPUT_DIR, 'src', 'main', 'res')
+    FileUtils.mkdir_p(output_dir)
+    downloader = Fastlane::Helper::GPDownloader.new(**WC_ANDROID)
+    downloader.download_all_locales(format: EXPORT_FMT::ANDROID) do |gp_locale, io|
+      locale = Locales.all.find(gp_locale)
+      Fastlane::Helper::Android::StringsFileWriter.write(dir: output_dir, locale: locale, io: io) unless locale.nil? # skip unknown locales that may be present in ZIP
+    end
+  end
+
+  def demo_ios_app_translations_bulk
+    output_dir = File.join(EXAMPLE_OUTPUT_DIR, 'Resources')
+    FileUtils.mkdir_p(output_dir)
+    downloader = Fastlane::Helper::GPDownloader.new(**WC_IOS)
+    downloader.download_all_locales(format: EXPORT_FMT::IOS) do |gp_locale, io|
+      locale = Locales.all.find(gp_locale)
+      Fastlane::Helper::Ios::StringsFileWriter.write(dir: output_dir, locale: locale, io: io) unless locale.nil? # skip unknown locales that may be present in ZIP
+    end
+  end
+
+  def demo_ios_app_translations_loop
+    output_dir = File.join(EXAMPLE_OUTPUT_DIR, 'Resources')
+    FileUtils.mkdir_p(output_dir)
+    downloader = Fastlane::Helper::GPDownloader.new(**WC_IOS)
+    Locales.mag16.each do |locale|
+      downloader.download_locale(gp_locale: locale.glotpress, format: EXPORT_FMT::IOS) do |io|
+        Fastlane::Helper::Ios::StringsFileWriter.write(dir: output_dir, locale: locale, io: io)
+      end
+    end
+  end
+
+  # Example Usages for Metadata
+
+  def demo_android_metadata_bulk
+    downloader = Fastlane::Helper::GPDownloader.new(host: DOTORG, project: 'apps/android/release-notes')
+    downloader.download_all_locales(format: EXPORT_FMT::JSON) do |gp_locale, io|
+      locale = Locales.mag16.find(gp_locale)
+      next unless Locale.valid?(locale, :playstore)
+
+      rules = FastlaneMetadataFilesWriter::MetadataRule.android_rules(version_name: '20.4', version_code: 1234)
+      translations = downloader.class.parse_json_export(io: io) # Convert odd GlotPress JSON export format to standard Hash
+
+      locale_dir = File.join(EXAMPLE_OUTPUT_DIR, 'fastlane', 'metadata', 'android', locale.playstore)
+      FastlaneMetadataFilesWriter.write(locale_dir: locale_dir, translations: translations, rules: rules) do |key|
+        # Example: if we find a non-standard key which ends up being a screenshot key, save under screenshots/ subdir.
+        # Otherwise, just ignore any other unknown key.
+        if key.start_with?('play_store_screenshot_')
+          FastlaneMetadataFilesWriter::MetadataRule.new(key, nil, File.join('screenshots', "#{key.delete_prefix('play_store_screenshot_')}.txt"))
+        end
+      end
+    end
+  end
+
+  def demo_android_metadata_loop
+    downloader = Fastlane::Helper::GPDownloader.new(host: DOTORG, project: 'apps/android/release-notes/')
+    Locales['fr', 'es'].each do |locale|
+      next unless Locale.valid?(locale, :playstore)
+      downloader.download_locale(gp_locale: locale.glotpress, format: EXPORT_FMT::JSON) do |io|
+        locale_dir = File.join(EXAMPLE_OUTPUT_DIR, 'fastlane', 'metadata', 'android', locale.playstore)
+        rules = FastlaneMetadataFilesWriter::MetadataRule.android_rules(version_name: '20.4', version_code: 1234)
+        translations = downloader.class.parse_json_export(io: io) # Convert odd GlotPress JSON export format to standard Hash
+        puts "Writing to #{locale_dir}..."
+        FastlaneMetadataFilesWriter.write(locale_dir: locale_dir, translations: translations, rules: rules)
+      end
+    end
+  end
+
+  def demo_ios_metadata_loop
+    downloader = Fastlane::Helper::GPDownloader.new(host: DOTORG, project: 'apps/ios/release-notes/')
+    Locales.mag16.each do |locale|
+      next unless Locale.valid?(locale, :appstore)
+      downloader.download_locale(gp_locale: locale.glotpress, format: EXPORT_FMT::JSON) do |io|
+        locale_dir = File.join(EXAMPLE_OUTPUT_DIR, 'fastlane', 'metadata', locale.appstore)
+        rules = FastlaneMetadataFilesWriter::MetadataRule.ios_rules(version_name: '20.4')
+        translations = downloader.class.parse_json_export(io: io) # Convert odd GlotPress JSON export format to standard Hash
+        FastlaneMetadataFilesWriter.write(locale_dir: locale_dir, translations: translations, rules: rules)
+      end
+    end
+  end
+end