Merge pull request creativecommons#516 from creativecommons/bin-scripts

Improve helper scripts

Author: TimidRobot

.github/workflows/check-python-versions.yml

@@ -16,4 +16,4 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Run script to check Python versions
-      run: ./dev/check_python_versions.sh
+      run: ./bin/check_python_versions.sh

README.md

@@ -129,7 +129,7 @@ repository (where `manage.py` is).
 6. Initialize data.
 Open a separate terminal tab, and in the same directory, run:
     ```shell
-    ./dev/init_data.sh
+    ./bin/init_data.sh
     ```
     1. Deletes database (which may not yet exist)
     2. Initializes database
@@ -215,26 +215,52 @@ Repository](#data-repository), above).
 #### Static Files Process
 
 This process will write the HTML files in the cc-legal-tools-data clone
-directory under `docs/`. It will not commit the changes (`--nogit`) and will
-not push any commits (`--nopush` is implied by `--nogit`).
+directory under `docs/`. By default it doesn't commit any changes.
 
 1. Ensure the [Data Repository](#data-repository), above, is in place
 2. Ensure [Docker Compose Setup](#docker-compose-setup), above, is complete
-3. Delete the contents of the `docs/` directory and then recreate/copy the
-   static files it should contain:
+3. Run the helper script:
     ```shell
-    docker compose exec app ./manage.py publish -v2
+    ./bin/publish.sh
     ```
+    - Example output:
+        ```
+        # Django publish                                                       19:36:20
+        INFO 17:36:21 Checking legal code titles
+        INFO 17:36:23 Purging output_dir: /home/cc/cc-legal-tools-data/docs
+        INFO 17:36:31 Collecting static files
+        66 static files copied to '/home/cc/cc-legal-tools-app/tmp/public/static', 158 unmodified.
+        INFO 17:36:31 Writing robots.txt
+        INFO 17:36:31 Copying WordPress content files
+        INFO 17:36:31 Copying static cc-legal-tools files
+        INFO 17:36:31 Copying static RDF/XML files
+        INFO 17:36:31 Distilling index.rdf
+        INFO 17:36:33 Distilling images.rdf
+        INFO 17:36:33 Distilling ns.html
+        INFO 17:36:33 Copying plaintext legal code
+        INFO 17:36:33 Distilling dev index
+        INFO 17:36:34 Distilling lists
+        INFO 17:36:36 Distilling Licenses 4.0 deed HTML, legal code HTML, and RDF/XML
+        INFO 17:36:41 Distilling Licenses 3.0 deed HTML, legal code HTML, and RDF/XML
+        INFO 17:37:23 Distilling Licenses 2.5 deed HTML, legal code HTML, and RDF/XML
+        INFO 17:37:50 Distilling Licenses 2.1 deed HTML, legal code HTML, and RDF/XML
+        INFO 17:37:54 Distilling Licenses 2.0 deed HTML, legal code HTML, and RDF/XML
+        INFO 17:38:14 Distilling Licenses 1.0 deed HTML, legal code HTML, and RDF/XML
+        INFO 17:38:20 Distilling Public Domain all deed HTML, legal code HTML, and RDF/XML
+        INFO 17:38:21 Writing Apache2 redirects configuration
+        ```
+       - (note that the laptop environment and docker environment had different
+         timezones, CEST and UTC)
 
 
 #### Publishing Changes to Git Repo
 
-When the site is deployed, to enable pushing and pulling the licenses data repo
+~~When the site is deployed, to enable pushing and pulling the licenses data repo
 with GitHub, create an SSH deploy key for the cc-legal-tools-data repo with
 write permissions, and put the private key file (not password protected)
 somewhere safe (owned by `www-data` if on a server), and readable only by its
 owner (0o400). Then in settings, make `TRANSLATION_REPOSITORY_DEPLOY_KEY` be
-the full path to that deploy key file.
+the full path to that deploy key file.~~ (not active nor current)
 
 
 #### Publishing Dependency Documentation
@@ -372,24 +398,36 @@ These resources are available for developing this tooling:
 ### Helper Scripts
 
 Best run before every commit:
-- `./dev/coverage.sh` - Run coverage tests and report
-- `./dev/tools.sh` - Run Python code tools (isort, black, flake8)
+- `./bin/coverage.sh` - Run coverage tests and report
+- `./bin/tools.sh` - Run Python code tools (isort, black, flake8)
 
 Run as needed:
-- `./dev/copy_theme.sh` - Copy the portions of
+- `./bin/copy_theme.sh` - Copy the portions of
   [creativecommons/vocabulary-theme][vocab-theme] needed for local development
   - Run after each new release of
     [creativecommons/vocabulary-theme][vocab-theme]
 
+Generate static files:
+- `./bin/publish.sh` - Run Django publish mangement command via Docker
+
+Django manage via Docker:
+- `./bin/manage.py` - Run Django management commands via Docker
+  - Example:
+    ```shell
+    ./bin/manage.py -h
+    ```
+
 Data management:
-- `./dev/dump_data.sh` - Dump Django application data
-- `./dev/init_data.sh` - :warning: Initialize Django application data
-- `./dev/load_data.sh` - Load Django application data
+- `./bin/dump_data.sh` - Dump Django application data
+- `./bin/init_data.sh` - :warning: Initialize Django application data
+- `./bin/load_data.sh` - Load Django application data
+
+
 
 Esoteric and dangerous:
-- `./dev/updatemessages.sh` - :warning: Run Django Management
-  nofuzzy_makemessages with helpful options (including excluding legalcode) and
-  compilemessages
+- `./bin/updatemessages.sh` - :warning: Run Django Management
+  `nofuzzy_makemessages` with helpful options (ex. excluding legalcode) and
+  `compilemessages`
 
 [vocab-theme]: https://github.com/creativecommons/vocabulary-theme
 

dev/20230901_norm_legacy_rdf.py renamed to bin/20230901_norm_legacy_rdf.py

@@ -1,10 +1,21 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Check if all specified Python versions match Pipfile
 #
+#### SETUP ####################################################################
+
+set -o errexit
 set -o errtrace
 set -o nounset
 
+# shellcheck disable=SC2154
+trap '_es=${?};
+    printf "${0}: line ${LINENO}: \"${BASH_COMMAND}\"";
+    printf " exited with a status of ${_es}\n";
+    exit ${_es}' ERR
+
+
+DIR_REPO="$(cd -P -- "${0%/*}/.." && pwd -P)"
 # https://en.wikipedia.org/wiki/ANSI_escape_code
 E0="$(printf    "\e[0m")"   # reset
 E91="$(printf   "\e[91m")"  # bright red foreground
@@ -13,6 +24,7 @@ E97="$(printf   "\e[97m")"  # bright white foreground
 E100="$(printf  "\e[100m")" # bright black (gray) background
 EXIT_STATUS=0
 
+#### FUNCTIONS ################################################################
 
 pyvercompare() {
     local _path=${1}
@@ -27,7 +39,7 @@ pyvercompare() {
     # compare Python version
     if [[ "${_path}" == 'Pipfile' ]]
     then
-        _status="✅  ${E92}athoritative Python version${E0}"
+        _status="✅  ${E92}authoritative Python version${E0}"
         PIPFILE_VER=${_ver}
     elif [[ "${_ver}" == "${PIPFILE_VER}" ]] \
         || [[ "${_ver}" == "python${PIPFILE_VER}" ]] \
@@ -46,11 +58,9 @@ pyvercompare() {
     echo
 }
 
+#### MAIN #####################################################################
 
-# Change directory to root directory of cc-legal-tools-app repository
-# (grandparent directory of this script)
-# shellcheck disable=SC2164
-cd "${0%/*}"/../
+cd "${DIR_REPO}"
 
 pyvercompare 'Pipfile' 'python_version =' '3'
 pyvercompare '.github/workflows/django-app-coverage.yml' 'python-version:' '2'

dev/20231009_concatenatemessages.sh renamed to bin/20231009_concatenatemessages.sh

@@ -1,4 +1,10 @@
-#!/bin/bash
+#!/usr/bin/env bash
+#
+# Copy vocabulary-theme files from a normal checkout (not a releaes tag
+# checkout, which has a different structure)
+#
+#### SETUP ####################################################################
+
 set -o errexit
 set -o errtrace
 set -o nounset
@@ -9,17 +15,17 @@ trap '_es=${?};
     printf " exited with a status of ${_es}\n";
     exit ${_es}' ERR
 
+DIR_REPO="$(cd -P -- "${0%/*}/.." && pwd -P)"
 # https://en.wikipedia.org/wiki/ANSI_escape_code
 E0="$(printf "\e[0m")"        # reset
-E1="$(printf "\e[1m")"        # bold
 E9="$(printf "\e[9m")"        # strike (not supported in Terminal.app)
 E30="$(printf "\e[30m")"      # black foreground
 E31="$(printf "\e[31m")"      # red foreground
-E33="$(printf "\e[33m")"      # yellow foreground
 E90="$(printf "\e[90m")"      # bright black (gray) foreground
 E97="$(printf "\e[97m")"      # bright white foreground
 E100="$(printf "\e[100m")"    # bright black (gray) background
 E107="$(printf "\e[107m")"    # bright white background
+# shellcheck disable=SC2016
 README='# Dev theme files
 
 This directory is only used in the Django and GitHub development environments.
@@ -42,9 +48,8 @@ THEME_DIR=''
 
 #### FUNCTIONS ################################################################
 
-
 copy_vocabulary_theme_files() {
-    header 'Copying necessary files from vocabulary-themes'
+    print_header 'Copying necessary files from vocabulary-themes'
     print_var REPO_DIR
     print_var STATIC_THEME_DIR | sed -e"s#${REPO_DIR}#.#"
     {
@@ -56,9 +61,8 @@ copy_vocabulary_theme_files() {
     echo
 }
 
-
 create_static_theme_dirs() {
-    header 'Creating necessary static theme directories'
+    print_header 'Creating necessary static theme directories'
     print_var REPO_DIR
     print_var STATIC_DIR | sed -e"s#${REPO_DIR}#.#"
     {
@@ -67,26 +71,23 @@ create_static_theme_dirs() {
     echo
 }
 
-
 create_wp_content_readme() {
-    header 'Creating wp-content README.md'
+    print_header 'Creating wp-content README.md'
     print_var REPO_DIR
     print_var STATIC_DIR | sed -e"s#${REPO_DIR}#.#"
     echo "${README}" > "${STATIC_DIR}/wp-content/README.md"
     echo "${E90}STATIC_DIR${E0}/wp-content/README.md"
     echo
 }
 
-
 error_exit() {
     # Echo error message and exit with error
     echo -e "${E31}ERROR:${E0} ${*}" 1>&2
     exit 1
 }
 
-
 get_vocabulary_theme_dir() {
-    header 'Getting vocabulary-theme dir'
+    print_header 'Getting vocabulary-theme dir'
     if ! THEME_DIR="$(cd -P -- \
         "${REPO_DIR}"/../vocabulary-theme 2> /dev/null \
         && pwd -P)" || ! [[ -d "${THEME_DIR}" ]]
@@ -100,47 +101,44 @@ get_vocabulary_theme_dir() {
     echo
 }
 
-
-header() {
+print_header() {
     # Print 80 character wide black on white heading with time
     printf "${E30}${E107} %-71s$(date '+%T') ${E0}\n" "${@}"
 }
 
-
 print_key_val() {
     printf "${E97}${E100}%18s${E0} %s\n" "${1}:" "${2}"
 }
 
-
 print_var() {
     print_key_val "${1}" "${!1}"
 }
 
-
 purge_static_theme_dir() {
-    header 'Purging existing static theme directories'
+    print_header 'Purging existing static theme directories'
     print_var REPO_DIR
     print_var STATIC_DIR | sed -e"s#${REPO_DIR}#.#"
     print_var STATIC_THEME_DIR | sed -e"s#${REPO_DIR}#.#"
     {
-        rm -f -r -v "${STATIC_THEME_DIR}/"*
+        rm -f -r -v "${STATIC_THEME_DIR:?}/"*
     } | sed \
         -e"s#${STATIC_THEME_DIR}#${E90}${E9}STATIC_THEME_DIR${E0}${E9}#" \
         -e"s#\$#${E0}#"
     print_var REPO_DIR
     print_var STATIC_DIR | sed -e"s#${REPO_DIR}#.#"
     print_var STATIC_THEME_DIR | sed -e"s#${REPO_DIR}#.#"
     {
-        rm -f -r -v "${STATIC_DIR}/wp-content"
+        rm -f -r -v "${STATIC_DIR:?}/wp-content"
     } | sed \
         -e"s#${STATIC_DIR}#${E90}${E9}STATIC_DIR${E0}${E9}#" \
         -e"s#\$#${E0}#"
     echo
 }
 
-
 #### MAIN #####################################################################
 
+cd "${DIR_REPO}"
+
 get_vocabulary_theme_dir
 purge_static_theme_dir
 create_static_theme_dirs

dev/20231129_modify_deeds_for_footnotes.sh renamed to bin/20231129_modify_deeds_for_footnotes.sh

@@ -1,11 +1,13 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Run coverage tests and report
 #
 # This script passes all arguments to Coverage Tests. For example, it can be
 # called from the cc-legal-tools-app directory like so:
 #
 #     ./dev/coverage.sh --failfast
+#
+#### SETUP ####################################################################
 
 set -o errexit
 set -o errtrace
@@ -50,6 +52,7 @@ docker compose exec app coverage erase --debug=dataio
 echo
 
 print_header 'Coverage tests'
+# shellcheck disable=SC2068
 docker compose exec app coverage run --debug=pytest \
     manage.py test --noinput --parallel 4 ${@:-} \
     || exit

dev/20250210_update_legacy_cc0.sh renamed to bin/20250210_update_legacy_cc0.sh

@@ -1,7 +1,9 @@
-#!/bin/bash
+#!/usr/bin/env bash
 #
 # Dump Django application data
 #
+#### SETUP ####################################################################
+
 set -o errexit
 set -o errtrace
 set -o nounset
@@ -12,6 +14,8 @@ trap '_es=${?};
     printf " exited with a status of ${_es}\n";
     exit ${_es}' ERR
 
+DATA_FILE='../cc-legal-tools-data/config/app_data.yaml'
+DIR_REPO="$(cd -P -- "${0%/*}/.." && pwd -P)"
 # https://en.wikipedia.org/wiki/ANSI_escape_code
 E0="$(printf "\e[0m")"        # reset
 E30="$(printf "\e[30m")"      # black foreground
@@ -20,7 +24,6 @@ E107="$(printf "\e[107m")"    # bright white background
 
 #### FUNCTIONS ################################################################
 
-
 check_docker() {
     local _msg
     if ! docker compose exec app true 2>/dev/null; then
@@ -30,29 +33,27 @@ check_docker() {
     fi
 }
 
-
 error_exit() {
     # Echo error message and exit with error
     echo -e "${E31}ERROR:${E0} ${*}" 1>&2
     exit 1
 }
 
-
 print_header() {
     # Print 80 character wide black on white heading with time
     printf "${E30}${E107}# %-70s$(date '+%T') ${E0}\n" "${@}"
 }
 
-
 #### MAIN #####################################################################
 
+cd "${DIR_REPO}"
+
 check_docker
-print_header 'Export data (LegalCode and Tool models)'
-data_file='../cc-legal-tools-data/config/app_data.yaml'
+print_header 'Django dumpdata - Export LegalCode and Tool models'
 docker compose exec app ./manage.py dumpdata \
     --format yaml \
     --indent 2 \
-    --output "${data_file}" \
+    --output "${DATA_FILE}" \
     legal_tools.LegalCode legal_tools.Tool
-du -h "${data_file}"
+du -h "${DATA_FILE}"
 echo