From 936f84184179b6dcd4567564581f7975b3b19ca3 Mon Sep 17 00:00:00 2001 From: Terry Geng Date: Mon, 13 Jul 2020 10:52:47 +0800 Subject: [PATCH 1/2] fix(CI): Check if PR exists before push. --- .drone.yml | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/.drone.yml b/.drone.yml index 5b7c430..b8f7d06 100644 --- a/.drone.yml +++ b/.drone.yml @@ -2,11 +2,19 @@ kind: pipeline type: docker name: deployement +volumes: + - name: repo + host: + path: /media/raid5/data/packages/repos/apt/botamusique/ + steps: - name: build-web image: node commands: - (cd web && npm install && npm run build) + when: + event: + - push - name: push-translation image: python:3 @@ -29,6 +37,9 @@ steps: commands: - pip3 install jinja2 - (cd templates/ && ./translate.py) + when: + event: + - push - name: deploy-testing image: debian @@ -111,11 +122,6 @@ steps: event: - tag -volumes: - - name: repo - host: - path: /media/raid5/data/packages/repos/apt/botamusique/ - trigger: event: exclude: @@ -124,7 +130,7 @@ trigger: --- kind: pipeline type: docker -name: cron +name: translation steps: - name: fetch-translation image: debian @@ -141,23 +147,23 @@ steps: - | if git fetch origin bot-traduora; then git branch bot-traduora FETCH_HEAD + CREATE_PR=false else git branch bot-traduora + CREATE_PR=true fi - git checkout bot-traduora - (cd lang/ && ./sync_translation.py --client $TRADUORA_R_CLIENT --secret $TRADUORA_R_SECRET --fetch) - git add lang/* - git status - | - if (GIT_COMMITTER_NAME='Traduora Bot' GIT_COMMITTER_EMAIL='noreply@azlux.fr' git commit -m 'Bot: Update translation' --author "Traduora Bot "); then - git remote set-url origin https://azlux:$GITHUB_API@github.com/azlux/botamusique/ + if GIT_COMMITTER_NAME='Traduora Bot' GIT_COMMITTER_EMAIL='noreply@azlux.fr' git commit -m 'Bot: Update translation' --author "Traduora Bot "; then git push origin bot-traduora sleep 2 - GITHUB_USER="azlux" GITHUB_TOKEN="$GITHUB_API" hub pull-request -m "Bot: TRADUORA Update" + if $CREATE_PR; then GITHUB_USER="azlux" GITHUB_TOKEN="$GITHUB_API" hub pull-request -m "Bot: TRADUORA Update"; fi fi trigger: event: - - cron - cron: - - auto-fetch-lang + include: + - push From 8844d088570a5e87bf1f191c22d5b5fe0909f183 Mon Sep 17 00:00:00 2001 From: Terry Geng Date: Mon, 13 Jul 2020 20:11:26 +0800 Subject: [PATCH 2/2] refactor(CI): Move scripts into scripts/ --- .drone.yml | 55 ++++++++----------- scripts/commit_new_translation.sh | 21 +++++++ {lang => scripts}/sync_translation.py | 12 +++- .../translate_templates.py | 31 ++++++++--- 4 files changed, 78 insertions(+), 41 deletions(-) create mode 100755 scripts/commit_new_translation.sh rename {lang => scripts}/sync_translation.py (89%) rename templates/translate.py => scripts/translate_templates.py (57%) diff --git a/.drone.yml b/.drone.yml index b8f7d06..88f76cf 100644 --- a/.drone.yml +++ b/.drone.yml @@ -25,7 +25,7 @@ steps: from_secret: TRADUORA_W_SECRET commands: - pip3 install requests - - (cd lang && ./sync_translation.py --client $TRADUORA_W_CLIENT --secret $TRADUORA_W_SECRET --push) + - ./scripts/sync_translation.py --lang_dir lang/ --client $TRADUORA_W_CLIENT --secret $TRADUORA_W_SECRET --push when: branch: - master @@ -36,7 +36,7 @@ steps: image: python:3 commands: - pip3 install jinja2 - - (cd templates/ && ./translate.py) + - ./scripts/translate_templates.py --lang-dir lang/ --template-dir templates/ when: event: - push @@ -82,14 +82,8 @@ steps: - name: deploy-stable image: debian - environment: - TRADUORA_W_CLIENT: - from_secret: TRADUORA_W_CLIENT - TRADUORA_W_SECRET: - from_secret: TRADUORA_W_SECRET commands: - apt-get -qq update && apt-get -qq install jq curl git pandoc python3-requests > /dev/null - - (cd lang && ./sync_translation.py --client $TRADUORA_W_CLIENT --secret $TRADUORA_W_SECRET --push) - sed -i 's/target_version = git/target_version = stable/' configuration.default.ini - git fetch --tags - version=$(git describe --abbrev=0 --tags) @@ -143,27 +137,26 @@ steps: from_secret: GITHUB_API commands: - apt update && apt install -y git python3-requests hub - - git remote set-url origin https://azlux:$GITHUB_API@github.com/azlux/botamusique/ - - | - if git fetch origin bot-traduora; then - git branch bot-traduora FETCH_HEAD - CREATE_PR=false - else - git branch bot-traduora - CREATE_PR=true - fi - - git checkout bot-traduora - - (cd lang/ && ./sync_translation.py --client $TRADUORA_R_CLIENT --secret $TRADUORA_R_SECRET --fetch) - - git add lang/* - - git status - - | - if GIT_COMMITTER_NAME='Traduora Bot' GIT_COMMITTER_EMAIL='noreply@azlux.fr' git commit -m 'Bot: Update translation' --author "Traduora Bot "; then - git push origin bot-traduora - sleep 2 - if $CREATE_PR; then GITHUB_USER="azlux" GITHUB_TOKEN="$GITHUB_API" hub pull-request -m "Bot: TRADUORA Update"; fi - fi + - SOURCE_DIR=$(pwd) ./scripts/commit_new_translation.sh + when: + event: + include: + - push -trigger: - event: - include: - - push + - name: fetch-translation-cron + image: debian + environment: + TRADUORA_R_CLIENT: + from_secret: TRADUORA_R_CLIENT + TRADUORA_R_SECRET: + from_secret: TRADUORA_R_SECRET + GITHUB_API: + from_secret: GITHUB_API + commands: + - apt update && apt install -y git python3-requests hub + - SOURCE_DIR=$(pwd) ./scripts/commit_new_translation.sh + when: + event: + - cron + cron: + - auto-fetch-lang diff --git a/scripts/commit_new_translation.sh b/scripts/commit_new_translation.sh new file mode 100755 index 0000000..9700b20 --- /dev/null +++ b/scripts/commit_new_translation.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +git remote set-url origin https://azlux:$GITHUB_API@github.com/azlux/botamusique/ + +if git fetch origin bot-traduora; then + git branch bot-traduora FETCH_HEAD + CREATE_PR=false +else + git branch bot-traduora + CREATE_PR=true +fi +git checkout bot-traduora +$SOURCE_DIR/scripts/sync_translation.py --lang_dir $SOURCE_DIR/lang/ --client $TRADUORA_R_CLIENT --secret $TRADUORA_R_SECRET --fetch +git add lang/* +git status + +if GIT_COMMITTER_NAME='Traduora Bot' GIT_COMMITTER_EMAIL='noreply@azlux.fr' git commit -m 'Bot: Update translation' --author "Traduora Bot "; then + git push origin bot-traduora + sleep 2 + if $CREATE_PR; then GITHUB_USER="azlux" GITHUB_TOKEN="$GITHUB_API" hub pull-request -m "Bot: TRADUORA Update"; fi +fi diff --git a/lang/sync_translation.py b/scripts/sync_translation.py similarity index 89% rename from lang/sync_translation.py rename to scripts/sync_translation.py index 67fb20b..87ecc57 100755 --- a/lang/sync_translation.py +++ b/scripts/sync_translation.py @@ -8,6 +8,8 @@ import requests base_url = "https://translate.azlux.fr/api/v1" project_id = "4aafb197-3282-47b3-a197-0ca870cf6ab2" +lang_dir = "" + def get_access_header(client, secret): data = {"grant_type": "client_credentials", @@ -39,7 +41,7 @@ def fetch_translation(r_client, r_secret): params = {'locale': lang_code, 'format': 'jsonnested'} r = requests.get(f"{base_url}/projects/{project_id}/exports", params=params, headers=headers) - with open(lang_code + ".json", "wb") as f: + with open(os.path.join(lang_dir, f"{lang_code}.json"), "wb") as f: f.write(r.content) @@ -47,7 +49,7 @@ def push_strings(w_client, w_secret): print("Pushing local translation files into the remote host...") headers = get_access_header(w_client, w_secret) - lang_files = os.listdir('.') + lang_files = os.listdir(lang_dir) lang_list = [] for lang_file in lang_files: match = re.search("([a-z]{2}_[A-Z]{2})\.json", lang_file) @@ -59,7 +61,7 @@ def push_strings(w_client, w_secret): params = {'locale': lang, 'format': 'jsonnested'} - files = {'file': open(lang + ".json", 'r')} + files = {'file': open(os.path.join(lang_dir, f"{lang}.json"), 'r')} r = requests.post(f"{base_url}/projects/{project_id}/imports", params=params, headers=headers, files=files) assert r.status_code == 200, f"Unable to push {lang} into remote host. {r.status_code}" @@ -69,6 +71,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description="Sync translation files with azlux's traduora server.") + parser.add_argument("--lang-dir", dest="lang_dir", + type=str, help="Directory of the lang files.") parser.add_argument("--client", dest="client", type=str, help="Client ID used to access the server.") parser.add_argument("--secret", dest="secret", @@ -81,6 +85,8 @@ if __name__ == "__main__": args = parser.parse_args() + lang_dir = args.lang_dir + if not args.client or not args.secret: print("Client ID and secret need to be provided!") exit(1) diff --git a/templates/translate.py b/scripts/translate_templates.py similarity index 57% rename from templates/translate.py rename to scripts/translate_templates.py index 86c6f86..6bd1983 100755 --- a/templates/translate.py +++ b/scripts/translate_templates.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 - +import argparse import os import json import re @@ -8,9 +8,12 @@ import jinja2 default_lang_dict = {} lang_dict = {} +lang_dir = "" +template_dir = "" + def load_lang(lang): - with open(f"../lang/{lang}.json", "r") as f: + with open(os.path.join(lang_dir, f"{lang}.json"), "r") as f: return json.load(f) @@ -22,11 +25,24 @@ def tr(option): string = default_lang_dict['web'][option] return string except KeyError: - raise KeyError("Missed strings in language file: '{string}'. ".format(string=option)) + raise KeyError("Missed strings in language file: '{string}'. " + .format(string=option)) if __name__ == "__main__": - html_files = os.listdir('.') + parser = argparse.ArgumentParser( + description="Populate html templates with translation strings.") + + parser.add_argument("--lang-dir", dest="lang_dir", + type=str, help="Directory of the lang files.") + parser.add_argument("--template-dir", dest="template_dir", + type=str, help="Directory of the template files.") + + args = parser.parse_args() + lang_dir = args.lang_dir + template_dir = args.template_dir + + html_files = os.listdir(template_dir) for html_file in html_files: match = re.search("(.+)\.template\.html", html_file) if match is None: @@ -34,10 +50,10 @@ if __name__ == "__main__": print(f"Populating {html_file} with translations...") basename = match[1] - with open(html_file, "r") as f: + with open(os.path.join(template_dir, f"{html_file}"), "r") as f: html = f.read() - lang_files = os.listdir('../lang') + lang_files = os.listdir(lang_dir) lang_list = [] default_lang_dict = load_lang("en_US") @@ -53,6 +69,7 @@ if __name__ == "__main__": print(f" - Populating {lang}...") lang_dict = load_lang(lang) - with open(f"{basename}.{lang}.html", "w") as f: + with open(os.path.join(template_dir, f"{basename}.{lang}.html"), + "w") as f: f.write(template.render(tr=tr)) print("Done.")