csv_to_content.py

#!/usr/bin/python3
# Note: use python3 for direct support of utf-8 strings

# This script requires python3 polib
# (sudo apt-get install python3-polib)

# To use this script, perform the following procedure:
# 1. Export spreadsheet from master content doc
#    and save in this folder as content.csv
# 2. Run this script to process content.csv
#    and update content/Default/apps/content.json
#    and .po files in the po directory
#    (Note that this does not update the other files
#    that are normally generated by unzip_content.py,
#    including the desktop files and the .pot file)
# 3. In the App Store sectin of the CMS, use the Import JSON
#    function to upload content/Default/apps/content.json
# 4. Generate Package and download the resulting appstore.zip
# 5. Move or copy appstore.zip to this folder
# 6. ./unzip_content.py
# 7. ./autogen.sh
# 8. make
# 9. make dist
# 10. tx push -s
# 11. tx push -t -f
#     Be careful -- the '-f' overwrites all existing translations!
#     If in doubt, you might want to pull the translation from Transifex
#     and commit them to git before starting this process.
# 12. tx pull -f
#     (Although not strictly necessary, this sets up the formatting
#     of the .po files to match subsequent pulls from Transifex)
# 13. ./unzip_content.py
#     (This second run regenerates files with the translations;
#     perhaps not strictly necesary, but let's follow the normal
#     process just to be sure)
# 14. Add and commit all changes to git
# 15. Re-publish any modified app bundles / bundle metadata
#     to the app server

import csv
import json
import os
import polib

CONTENT_JSON = 'content/Default/apps/content.json'
CONTENT_CSV = 'content.csv'
PO_DIR = 'po'
LANGS = {'en': 'GLOBAL',
         'es': 'SPANISH',
         'pt_BR': 'PORTUGUESE'}

if __name__ == '__main__':

    # Load the content json data from file
    with open(CONTENT_JSON) as json_file:
        json_data = json.load(json_file)

    # Open the input csv reader
    with open(CONTENT_CSV, newline='') as in_file:
        in_lines = in_file.readlines()
    csv_reader = csv.reader(in_lines)

    # Read and parse the three-row header
    header1 = csv_reader.__next__()
    header2 = csv_reader.__next__()
    header3 = csv_reader.__next__()
    lang_idx = {}
    for lang in LANGS:
        lang_idx[lang] = header1.index(LANGS[lang])
    appid_idx = header2.index('App Id')
    title_idx = lang_idx['en']
    num_cols = len(header2)
    empty_row = [''] * num_cols

    # Open po files
    po = {}
    for lang in LANGS:
        if lang != 'en':
            po_file = os.path.join(PO_DIR, lang + '.po')
            po[lang] = polib.pofile(po_file)

    # For now, we don't support any new-lines within the description,
    # so replace any newlines with spaces
    # Double-quotes are also problematic, so replace them
    # with single-quotes
    def sanitize(val):
        return val.strip().replace('\n', ' ').replace('"', "'")

    def update_row(csv_row, json_row):
        en_title = sanitize(csv_row[lang_idx['en']])
        en_subtitle = sanitize(csv_row[lang_idx['en'] + 1])
        en_description = sanitize(csv_row[lang_idx['en'] + 2])
        json_row['title'] = en_title
        json_row['subtitle'] = en_subtitle
        json_row['description'] = en_description

        def translate(lang, val, translation, msgctxt):
            entry = po[lang].find(val)
            if entry:
                entry.msgstr = translation
            else:
                entry = polib.POEntry(msgid=val,
                                      msgstr=translation,
                                      msgctxt=msgctxt)
                po[lang].append(entry)

        for lang in LANGS:
            if lang != 'en':
                title_idx = lang_idx[lang]
                subtitle_idx = lang_idx[lang] + 1
                description_idx = lang_idx[lang] + 2
                title = sanitize(csv_row[title_idx])
                subtitle = sanitize(csv_row[subtitle_idx])
                description = sanitize(csv_row[description_idx])
                translate(lang, en_title, title, 'title')
                translate(lang, en_subtitle, subtitle, 'subtitle')
                translate(lang, en_description, description, 'description')

    # Process each row of the input csv
    for csv_row in csv_reader:
        app_id = csv_row[appid_idx]
        if app_id:
            # Csv has app id: search for it in json
            for json_row in json_data:
                if json_row['application-id'] == app_id:
                    # Found a match in json
                    break
            else:
                # No match in json: ignore the row
                print('Warning: app id %s not found in content.json' % app_id)
                json_row = None
        else:
            # Csv has no app id: ignore the row
            print('Warning: no app id specified for %s' % csv_row[title_idx])
            json_row = None

        # If matching json content found, update it per the csv row
        if json_row:
            update_row(csv_row, json_row)

    # Re-write the content json file with any modifications
    with open(CONTENT_JSON, 'w') as json_file:
        json.dump(json_data, json_file, indent=2, sort_keys=True)

    # Re-write the po files with any modifications
    for lang in LANGS:
        if lang != 'en':
            po_file = os.path.join(PO_DIR, lang + '.po')
            po[lang].save(po_file)