From 46ac68822c89214d6a75dc800c3f3419f4b6bcb2 Mon Sep 17 00:00:00 2001 From: Patrick Marie Date: Wed, 2 Oct 2024 23:14:50 +0200 Subject: [PATCH] initial commit --- .github/workflows/backup.yaml | 40 +++++++ .github/workflows/tests.yaml | 17 +++ README.md | 49 ++++++++ scripts/backup.sh | 109 ++++++++++++++++++ scripts/tests.sh | 26 +++++ .../README.md | 78 +++++++++++++ .../README.md | 84 ++++++++++++++ .../test/test.txt | 1 + .../README.md | 78 +++++++++++++ .../doc.txt | 1 + .../doc.txt | 2 + .../README.md | 89 ++++++++++++++ .../README.md | 91 +++++++++++++++ .../README.md | 100 ++++++++++++++++ .../project/project1.txt | 1 + .../README.md | 92 +++++++++++++++ .../README.md | 78 +++++++++++++ .../doc.txt | 1 + 18 files changed, 937 insertions(+) create mode 100644 .github/workflows/backup.yaml create mode 100644 .github/workflows/tests.yaml create mode 100644 README.md create mode 100644 scripts/backup.sh create mode 100644 scripts/tests.sh create mode 100644 tests/tests_cases/1/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md create mode 100644 tests/tests_cases/10/005a9f74f588073a321f45e7f527b0f09df6b3c7/README.md create mode 100644 tests/tests_cases/10/21e5331d1c0256701bb90cf017e519d54a88f618/test/test.txt create mode 100644 tests/tests_cases/10/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md create mode 100644 tests/tests_cases/10/46fe26c9dcf2354a0ed3f304ed6818de9606f7b5/doc.txt create mode 100644 tests/tests_cases/10/47998b5317e66b3bd456cfb07268c93e223704f2/doc.txt create mode 100644 tests/tests_cases/10/59e9ddff4dea9f744a2b84f6183c84afae559279/README.md create mode 100644 tests/tests_cases/10/729e5d2addf0a3566e454ddccf549041f2e9ab17/README.md create mode 100644 tests/tests_cases/10/77feffe3f6ccdf44a5f4150b3258e1a745e57807/README.md create mode 100644 tests/tests_cases/10/7c5aebc1feeef4eaf19083019547457b8cf3fc3d/project/project1.txt create mode 100644 tests/tests_cases/10/c37c77fbf9d12418c83e1510fb0bdece8bd2e11c/README.md create mode 100644 tests/tests_cases/2/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md create mode 100644 tests/tests_cases/2/46fe26c9dcf2354a0ed3f304ed6818de9606f7b5/doc.txt diff --git a/.github/workflows/backup.yaml b/.github/workflows/backup.yaml new file mode 100644 index 0000000..b1dc5da --- /dev/null +++ b/.github/workflows/backup.yaml @@ -0,0 +1,40 @@ +name: Run backup + +on: + schedule: + - cron: '30 * * * 1-5' + push: + branches: + - main + workflow_dispatch: + inputs: + num: + description: 'Number of commit to backup (0: unlimited)' + required: true + default: '10' + +jobs: + backup-and-push: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set num default value + run: | + NUM=${{ github.event.inputs.num }} + echo "NUM=${NUM:-10}" >> $GITHUB_ENV + + - name: Run backup script + run: | + /bin/bash scripts/backup.sh -r https://github.com/descartes-underwriting/devops-technical-test-data.git -b 01-01-2022-test -n ${{ env.NUM }} + /bin/bash scripts/backup.sh -r https://github.com/descartes-underwriting/devops-technical-test-data.git -b main -d $(pwd)/data_main -n ${{ env.NUM }} + + - name: Commit and push changes + run: | + git config --local user.email "action@github.com" + git config --local user.name "GitHub Action" + git add data data_main + git commit -m "Auto-update from backup job" || exit 0 + git push + diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml new file mode 100644 index 0000000..e2c5ecd --- /dev/null +++ b/.github/workflows/tests.yaml @@ -0,0 +1,17 @@ +name: Tests + +on: + push: + branches: + - main + +jobs: + tests: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Run tests + run: | + /bin/bash scripts/tests.sh diff --git a/README.md b/README.md new file mode 100644 index 0000000..dab68af --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# backup.sh ~ descartes underwriting technical test for devops + +The script takes as input a repository url, a branch, a destination directory and optionally a number of commit to backup, verbose mode, debug mode. + +It checkouts a git repository, list all commits in a given branch (from oldest to most recent), and for each commit sha checks if it was already backuped (check for ${DATA_DIR}/ directory), and if not, checks out all files mentionned in commit and dump their state at given commit into the per commit given directory. + +The included workflow that actually performs the backup can be manually run through github UI. + +## Usage + +```sh +$ /bin/bash scripts/backup.sh -r -b -d [-n ] [-v] [-x] +``` + +Ex: + +```sh +$ /bin/bash scripts/backup.sh -r https://github.com/descartes-underwriting/devops-technical-test-data.git -b 01-01-2022-test -d $(pwd)/data -n 5 +Cloning into 'devops-technical-test-data'... +remote: Enumerating objects: 21265, done. +... +new commit: 282180fe7e5d9cbf297f2f0ef813cffe60ce2328 +new commit: 46fe26c9dcf2354a0ed3f304ed6818de9606f7b5 +new commit: 21e5331d1c0256701bb90cf017e519d54a88f618 +new commit: 47998b5317e66b3bd456cfb07268c93e223704f2 +new commit: 7c5aebc1feeef4eaf19083019547457b8cf3fc3d +done: 5 + +$ ls -l data/ +total 0 +drwxr-xr-x 1 patrick chicac 8 Jan 1 09:28 21e5331d1c0256701bb90cf017e519d54a88f618 +drwxr-xr-x 1 patrick chicac 18 Jan 1 09:28 282180fe7e5d9cbf297f2f0ef813cffe60ce2328 +drwxr-xr-x 1 patrick chicac 14 Jan 1 09:28 46fe26c9dcf2354a0ed3f304ed6818de9606f7b5 +drwxr-xr-x 1 patrick chicac 14 Jan 1 09:28 47998b5317e66b3bd456cfb07268c93e223704f2 +drwxr-xr-x 1 patrick chicac 14 Jan 1 09:28 7c5aebc1feeef4eaf19083019547457b8cf3fc3d + +$ +``` + +Check `.github/workflows/backup.yaml` for usage sample. + +## Tests + +A sample test script exists to verify basic use case of backup'ing a limited number of commits. The `scripts/test.sh` runs a couple of backups and run a `diff -r` against a manually verified backup included in the repository. A workflow runs tests on push. + +## Not covered / Improvements ideas + +- Deleted files in commits; The backup.sh script does not cover deleted files; +- Keep track of latest commit backuped to allow iterative backups. As for now, the script checks all commits (which is not really efficient); However, if latest commit backup is tracked and this commit to be overwritten by a force push, it will be eventually required to re-do the whole backup. diff --git a/scripts/backup.sh b/scripts/backup.sh new file mode 100644 index 0000000..915e644 --- /dev/null +++ b/scripts/backup.sh @@ -0,0 +1,109 @@ +#!/bin/sh + +set -e + +REPOSITORY=https://github.com/descartes-underwriting/devops-technical-test-data.git +BRANCH=main +DATA_DIR=$(pwd)/data +MAX_NUM=0 +VERBOSE=0 + +while getopts ":r:b:d:vxn:" opt +do + case ${opt} in + r) + REPOSITORY=${OPTARG} + ;; + b) + BRANCH=${OPTARG} + ;; + d) + DATA_DIR=${OPTARG} + ;; + n) + MAX_NUM=${OPTARG} + ;; + v) + VERBOSE=1 + ;; + x) + set -x + ;; + ?) + echo "Invalid option -${OPTARG}" + exit 1 + ;; + esac +done + +IFS=" +" + +GIT=git +GIT_OPTS=(-c core.quotepath=false) + +if test ! -d ${DATA_DIR} +then + mkdir -p ${DATA_DIR} +fi + +TMPDIR=$(mktemp -d) +cd ${TMPDIR} + +git clone ${REPOSITORY} && cd ./* + +# List all commit sha, from older to newer; +COMMIT_SHAS=$($GIT ${GIT_OPTS[@]} log --reverse --pretty=format:"%H" remotes/origin/${BRANCH}) + +NUM_ADDED=0 + +for COMMIT_SHA in ${COMMIT_SHAS} +do + if test -d ${DATA_DIR}/${COMMIT_SHA} + then + # commit already backup; skipping + continue + fi + + echo "new commit: ${COMMIT_SHA}" + + # There are malformed files names that are creating complex filenames to parse. + # Those malformed filenames are double quotes, so to remove quotes, -c core.quotepath=false + # and -z are used. sed 's/\x0//g' is removing the null byte + FILES=$($GIT ${GIT_OPTS[@]} show --pretty= --name-only -z ${COMMIT_SHA} | sed 's/\x0//g') + + if test -z ${FILES} + then + # merge commit, etc. There is no file here. + echo "No file was found in commit ${COMMIT_SHA}; skipping" + continue + fi + + TARGET_BACKUP_SHA=${DATA_DIR}/${COMMIT_SHA} + mkdir -p ${TARGET_BACKUP_SHA} + + for FILE in ${FILES} + do + TARGET_DIR=$(dirname ${FILE}) + mkdir -p ${TARGET_BACKUP_SHA}/${TARGET_DIR} + + if test ${VERBOSE} -eq 1 + then + echo "Writing ${TARGET_BACKUP_SHA}/${FILE}" + fi + + # ${FILE} contains path/to/file + $GIT ${GIT_OPTS[@]} show ${COMMIT_SHA}:${FILE} > ${TARGET_BACKUP_SHA}/${FILE} + done + + NUM_ADDED=$((NUM_ADDED + 1)) + + if test ${NUM_ADDED} -eq ${MAX_NUM} + then + echo "Max commit to backup reached; stopping." + break + fi +done + +echo "done: ${NUM_ADDED}" + diff --git a/scripts/tests.sh b/scripts/tests.sh new file mode 100644 index 0000000..354d1df --- /dev/null +++ b/scripts/tests.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +run_test() { + echo "Running backup on branch main; dest $1; num: $2..." + /bin/bash scripts/backup.sh -b main -d $1 -n $2 + echo + diff -r $1 tests/tests_cases/$2 + if test $? -ne 0 + then + echo "Test fail" + exit 1 + fi + echo "... done" + echo +} + +ROOTDIR=$(git rev-parse --show-toplevel) +TMPDIR=$(mktemp -d) + +cd ${ROOTDIR} + +run_test ${TMPDIR}/1 1 +run_test ${TMPDIR}/2 2 +run_test ${TMPDIR}/10 10 + +echo "All tests were OK" diff --git a/tests/tests_cases/1/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md b/tests/tests_cases/1/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md new file mode 100644 index 0000000..9ba167d --- /dev/null +++ b/tests/tests_cases/1/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md @@ -0,0 +1,78 @@ +# Descartes Underwriting + +## Context + +We wish to create a backup tool that will save only the last modified files of a storage unit. + +In our example, the storage unit is **not a bucket**. + +The storage unit is a git repository called `devops-technical-test-data`. + +The repository `devops-technical-test-data` is not frozen and will have new commits. + +Commits will be added to the `datestamp-test` branch on the `devops-technical-test-data` repository. + +## Task + +Develop a backup tool to save the modified files at each commit. + +### Submission + +Script and data should be saved on a private `descartes-backup-project` repository on your github account. + +Access should be granted to all members of the `descartes-underwriting` group: + + + +### Script + +Create a script to automate the backup process using open source software. + +The script should track the branch `datestamp-test` of the repository. + +The execution of the script should be carried with a github-action / gitlab-pipeline or any other git automated workflow. + +### Data + +The backup should store files in separate folders. + +The backup file structure should be based on the sha1 of the `devops-technical-test-data`. + +## File structure example + +For the following commits on the `devops-technical-test-data`: + +| SHA | OPERATION | +|-----|-----------| +| Commit_N | create readme.md | +| Commit_N+1 | create doc.txt | +| Commit_N+2 | create data/test/test.txt | +| Commit_N+3 | append text to ./doc.txt | +| Commit_N+4 | create test/project/project1.txt | + +The `candidate_backup_repository` should have + +```bash +$ tree . +. +├── .gitworkflow +│   └── workflows +│   └── my-lovely-workflow.yml +├── data +│   ├── N +│   │   └── readme.md +│   ├── N+1 +│   │   └── doc.txt +│   ├── N+2 +│   │   └── data +│   │   └── test +│   │   └── test.txt +│   ├── N+3 +│   │   └── doc.txt +│   └── N+4 +│   └── test +│   └── project +│   └── project1.txt +└── script + └── my-beautiful-script.best-language +``` diff --git a/tests/tests_cases/10/005a9f74f588073a321f45e7f527b0f09df6b3c7/README.md b/tests/tests_cases/10/005a9f74f588073a321f45e7f527b0f09df6b3c7/README.md new file mode 100644 index 0000000..a67c45f --- /dev/null +++ b/tests/tests_cases/10/005a9f74f588073a321f45e7f527b0f09df6b3c7/README.md @@ -0,0 +1,84 @@ +# Descartes Underwriting + +## Context + +We wish to create a backup tool that will save only the last modified files of a storage unit. + +In our example, the storage unit is **not a bucket**. + +The storage unit is the `DD-MM-YYYY-test` branch of the current `descartes-underwriting/devops-technical-test-data` git repository. + +## Property + +The `descartes-underwriting/devops-technical-test-data` repository is not frozen and will have new commits. + +Commits will be added to the `DD-MM-YYYY-test` branch multiple times every day. + +The `DD-MM-YYYY-test` branch name will be adapted using standard datetime convention eg: `01-01-2022-test` for the 1st of January 2022. + +## Task + +Develop a backup tool to save the modified files at each commit. + +### Submission + +Script and data should be saved on a private `candidate/descartes-backup-project` repository on your github account. + +Access should be granted to all members of the `descartes-underwriting` group: + + + +### Script + +Create a script to automate the backup process using open source software. + +The script should track the changes fo the branch `DD-MM-YYYY-test` of the `descartes-underwriting/devops-technical-test-data` repository. + +The execution of the script should be carried out with a github-action / gitlab-pipeline or any other tool automating git workflow on your git project. + +It is highly recommended to use a scheduling tool to execute the back up process. + +### Data + +The backup should store files in separate folders. + +The backup file structure should be based on the sha1 of the `descartes-underwriting/devops-technical-test-data`. + +## File structure example + +For the following commits on the `descartes-underwriting/devops-technical-test-data`: + +| SHA | OPERATION | +|-----|-----------| +| Commit_N | create readme.md | +| Commit_N+1 | create doc.txt | +| Commit_N+2 | create data/test/test.txt | +| Commit_N+3 | append text to ./doc.txt | +| Commit_N+4 | create test/project/project1.txt | + +The `candidate/descartes-backup-project` repository should have + +```bash +$ tree . +. +├── .gitworkflow +│   └── workflows +│   └── my-lovely-workflow.yml +├── data +│   ├── N +│   │   └── readme.md +│   ├── N+1 +│   │   └── doc.txt +│   ├── N+2 +│   │   └── data +│   │   └── test +│   │   └── test.txt +│   ├── N+3 +│   │   └── doc.txt +│   └── N+4 +│   └── test +│   └── project +│   └── project1.txt +└── script + └── my-beautiful-script.best-language +``` diff --git a/tests/tests_cases/10/21e5331d1c0256701bb90cf017e519d54a88f618/test/test.txt b/tests/tests_cases/10/21e5331d1c0256701bb90cf017e519d54a88f618/test/test.txt new file mode 100644 index 0000000..9f4b6d8 --- /dev/null +++ b/tests/tests_cases/10/21e5331d1c0256701bb90cf017e519d54a88f618/test/test.txt @@ -0,0 +1 @@ +This is a test file diff --git a/tests/tests_cases/10/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md b/tests/tests_cases/10/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md new file mode 100644 index 0000000..9ba167d --- /dev/null +++ b/tests/tests_cases/10/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md @@ -0,0 +1,78 @@ +# Descartes Underwriting + +## Context + +We wish to create a backup tool that will save only the last modified files of a storage unit. + +In our example, the storage unit is **not a bucket**. + +The storage unit is a git repository called `devops-technical-test-data`. + +The repository `devops-technical-test-data` is not frozen and will have new commits. + +Commits will be added to the `datestamp-test` branch on the `devops-technical-test-data` repository. + +## Task + +Develop a backup tool to save the modified files at each commit. + +### Submission + +Script and data should be saved on a private `descartes-backup-project` repository on your github account. + +Access should be granted to all members of the `descartes-underwriting` group: + + + +### Script + +Create a script to automate the backup process using open source software. + +The script should track the branch `datestamp-test` of the repository. + +The execution of the script should be carried with a github-action / gitlab-pipeline or any other git automated workflow. + +### Data + +The backup should store files in separate folders. + +The backup file structure should be based on the sha1 of the `devops-technical-test-data`. + +## File structure example + +For the following commits on the `devops-technical-test-data`: + +| SHA | OPERATION | +|-----|-----------| +| Commit_N | create readme.md | +| Commit_N+1 | create doc.txt | +| Commit_N+2 | create data/test/test.txt | +| Commit_N+3 | append text to ./doc.txt | +| Commit_N+4 | create test/project/project1.txt | + +The `candidate_backup_repository` should have + +```bash +$ tree . +. +├── .gitworkflow +│   └── workflows +│   └── my-lovely-workflow.yml +├── data +│   ├── N +│   │   └── readme.md +│   ├── N+1 +│   │   └── doc.txt +│   ├── N+2 +│   │   └── data +│   │   └── test +│   │   └── test.txt +│   ├── N+3 +│   │   └── doc.txt +│   └── N+4 +│   └── test +│   └── project +│   └── project1.txt +└── script + └── my-beautiful-script.best-language +``` diff --git a/tests/tests_cases/10/46fe26c9dcf2354a0ed3f304ed6818de9606f7b5/doc.txt b/tests/tests_cases/10/46fe26c9dcf2354a0ed3f304ed6818de9606f7b5/doc.txt new file mode 100644 index 0000000..225fcd6 --- /dev/null +++ b/tests/tests_cases/10/46fe26c9dcf2354a0ed3f304ed6818de9606f7b5/doc.txt @@ -0,0 +1 @@ +This is a doc diff --git a/tests/tests_cases/10/47998b5317e66b3bd456cfb07268c93e223704f2/doc.txt b/tests/tests_cases/10/47998b5317e66b3bd456cfb07268c93e223704f2/doc.txt new file mode 100644 index 0000000..af207fa --- /dev/null +++ b/tests/tests_cases/10/47998b5317e66b3bd456cfb07268c93e223704f2/doc.txt @@ -0,0 +1,2 @@ +This is a doc +Hello, world ! diff --git a/tests/tests_cases/10/59e9ddff4dea9f744a2b84f6183c84afae559279/README.md b/tests/tests_cases/10/59e9ddff4dea9f744a2b84f6183c84afae559279/README.md new file mode 100644 index 0000000..dff1451 --- /dev/null +++ b/tests/tests_cases/10/59e9ddff4dea9f744a2b84f6183c84afae559279/README.md @@ -0,0 +1,89 @@ +# Descartes Underwriting + +## Context + +We wish to create a backup tool that will save only the last modified files of a storage unit. + +In our example, the storage unit is **not a bucket**. + +The storage unit is the `DD-MM-YYYY-test` branch of the current `descartes-underwriting/devops-technical-test-data` git repository. + +## Property + +The `descartes-underwriting/devops-technical-test-data` repository is not frozen and will have new commits. + +Commits will be added to the `DD-MM-YYYY-test` branch multiple times every day. + +The `DD-MM-YYYY-test` branch name will be adapted using standard datetime convention eg: `01-01-2022-test` for the 1st of January 2022. + +## Task + +Develop a backup tool to save the modified files at each commit. + +### Submission + +Script and data should be saved on a private `candidate/descartes-backup-project` repository on your github account. + +Access should be granted to all members of the `descartes-underwriting` group: + + + +Especially: + +* +* + +### Script + +Create a script to automate the backup process using open source software. + +The script should track the changes fo the branch `DD-MM-YYYY-test` of the `descartes-underwriting/devops-technical-test-data` repository. + +The execution of the script should be carried out with a github-action / gitlab-pipeline or any other tool automating git workflow on your git project. + +It is highly recommended to use a scheduling tool to execute the back up process. + +### Data + +The backup should store files in separate folders. + +The backup file structure should be based on the sha1 of the `descartes-underwriting/devops-technical-test-data`. + +## File structure example + +For the following commits on the `descartes-underwriting/devops-technical-test-data`: + +| SHA | OPERATION | +|-----|-----------| +| Commit_N | create readme.md | +| Commit_N+1 | create doc.txt | +| Commit_N+2 | create data/test/test.txt | +| Commit_N+3 | append text to ./doc.txt | +| Commit_N+4 | create test/project/project1.txt | + +The `candidate/descartes-backup-project` repository should have + +```bash +$ tree . +. +├── .gitworkflow +│   └── workflows +│   └── my-lovely-workflow.yml +├── data +│   ├── N +│   │   └── readme.md +│   ├── N+1 +│   │   └── doc.txt +│   ├── N+2 +│   │   └── data +│   │   └── test +│   │   └── test.txt +│   ├── N+3 +│   │   └── doc.txt +│   └── N+4 +│   └── test +│   └── project +│   └── project1.txt +└── script + └── my-beautiful-script.best-language +``` diff --git a/tests/tests_cases/10/729e5d2addf0a3566e454ddccf549041f2e9ab17/README.md b/tests/tests_cases/10/729e5d2addf0a3566e454ddccf549041f2e9ab17/README.md new file mode 100644 index 0000000..e3e1571 --- /dev/null +++ b/tests/tests_cases/10/729e5d2addf0a3566e454ddccf549041f2e9ab17/README.md @@ -0,0 +1,91 @@ +# Descartes Underwriting + +## Context + +We wish to create a backup tool that will save only the last modified files of a storage unit. + +In our example, the storage unit is **not a bucket**. + +The storage unit is the `DD-MM-YYYY-test` branch of the current `descartes-underwriting/devops-technical-test-data` git repository. + +## Property + +The `descartes-underwriting/devops-technical-test-data` repository is not frozen and will have new commits. + +Commits will be added to the `DD-MM-YYYY-test` branch multiple times every day. + +The `DD-MM-YYYY-test` branch name will be adapted using standard datetime convention eg: `01-01-2022-test` for the 1st of January 2022. + +## Task + +Develop a backup tool to save the modified files at each commit. + +### Submission + +Script and data should be saved on a private `candidate/descartes-backup-project` repository on your github account. + +Access should be granted to all members of the `descartes-underwriting` group: + + + +Especially: + +* +* + +### Script + +Create a script to automate the backup process using open source software. + +The script should track the changes fo the branch `DD-MM-YYYY-test` of the `descartes-underwriting/devops-technical-test-data` repository. + +The execution of the script should be carried out with a github-action / gitlab-pipeline or any other tool automating git workflow on your git project. + +It is highly recommended to use a scheduling tool to execute the back up process. + +### Data + +The backup should store files in separate folders. + +The backup file structure should be based on the sha1 of the `descartes-underwriting/devops-technical-test-data`. + +Starting from the initial commit [282180fe7e5d9cbf297f2f0ef813cffe60ce2328](https://github.com/descartes-underwriting/devops-technical-test-data/commit/282180fe7e5d9cbf297f2f0ef813cffe60ce2328), all the history should be backup. + +## File structure example + +For the following commits on the `descartes-underwriting/devops-technical-test-data`: + +| SHA | OPERATION | +|-----|-----------| +| Commit_N | create readme.md | +| Commit_N+1 | create doc.txt | +| Commit_N+2 | create data/test/test.txt | +| Commit_N+3 | append text to ./doc.txt | +| Commit_N+4 | create test/project/project1.txt | + +The `candidate/descartes-backup-project` repository should have + +```bash +$ tree . +. +├── .gitworkflow +│   └── workflows +│   └── my-lovely-workflow.yml +├── data +│   ├── N +│   │   └── readme.md +│   ├── N+1 +│   │   └── doc.txt +│   ├── N+2 +│   │   └── data +│   │   └── test +│   │   └── test.txt +│   ├── N+3 +│   │   └── doc.txt +│   └── N+4 +│   └── test +│   └── project +│   └── project1.txt +└── script + └── my-beautiful-script.best-language +``` diff --git a/tests/tests_cases/10/77feffe3f6ccdf44a5f4150b3258e1a745e57807/README.md b/tests/tests_cases/10/77feffe3f6ccdf44a5f4150b3258e1a745e57807/README.md new file mode 100644 index 0000000..be0f4bf --- /dev/null +++ b/tests/tests_cases/10/77feffe3f6ccdf44a5f4150b3258e1a745e57807/README.md @@ -0,0 +1,100 @@ +# Descartes Underwriting + +## Context + +We wish to create a backup tool that will save only the last modified files of a storage unit. + +In our example, the storage unit is **not a bucket**. + +The storage unit is the `DD-MM-YYYY-test` branch of the current `descartes-underwriting/devops-technical-test-data` git repository. + +## Property + +The `descartes-underwriting/devops-technical-test-data` repository is not frozen and will have new commits. + +Commits will be added to the `DD-MM-YYYY-test` branch multiple times every day. + +The `DD-MM-YYYY-test` branch name will be adapted using standard datetime convention eg: `01-01-2022-test` for the 1st of January 2022. + +## Task + +Develop a backup tool to save the modified files at each commit. + +### Submission + +If something is not clear, you can ask questions to the recruiter. + +When submitting your project, your version should **not be draft** but complete and following best practices. + +The solution should be saved on a **private** `descartes-devops` repository on your github account. + +The solution should include: + +- source code +- test code + +When the final version is ready: + +1. Send an email to the recruiter indicating that you finished the project and sharing the url of the project +2. Grant access to: + +- +- +- + +### Script + +Create a script to automate the backup process using open source software. + +The script should track the changes fo the branch `DD-MM-YYYY-test` of the `descartes-underwriting/devops-technical-test-data` repository. + +The execution of the script should be carried out with a github-action / gitlab-pipeline or any other tool automating git workflow on your git project. + +It is highly recommended to use a scheduling tool to execute the back up process. + +### Data + +The backup should store files in separate folders. + +The backup file structure should be based on the sha1 of the `descartes-underwriting/devops-technical-test-data`. + +Starting from the initial commit [282180fe7e5d9cbf297f2f0ef813cffe60ce2328](https://github.com/descartes-underwriting/devops-technical-test-data/commit/282180fe7e5d9cbf297f2f0ef813cffe60ce2328), all the history should be backup. + +## File structure example + +For the following commits on the `descartes-underwriting/devops-technical-test-data`: + +| SHA | OPERATION | +|-----|-----------| +| Commit_N | create readme.md | +| Commit_N+1 | create doc.txt | +| Commit_N+2 | create data/test/test.txt | +| Commit_N+3 | append text to ./doc.txt | +| Commit_N+4 | create test/project/project1.txt | + +The `candidate/descartes-backup-project` repository should have + +```bash +$ tree . +. +├── .gitworkflow +│   └── workflows +│   └── my-lovely-workflow.yml +├── data +│   ├── N +│   │   └── readme.md +│   ├── N+1 +│   │   └── doc.txt +│   ├── N+2 +│   │   └── data +│   │   └── test +│   │   └── test.txt +│   ├── N+3 +│   │   └── doc.txt +│   └── N+4 +│   └── test +│   └── project +│   └── project1.txt +└── script + └── my-beautiful-script.best-language +``` diff --git a/tests/tests_cases/10/7c5aebc1feeef4eaf19083019547457b8cf3fc3d/project/project1.txt b/tests/tests_cases/10/7c5aebc1feeef4eaf19083019547457b8cf3fc3d/project/project1.txt new file mode 100644 index 0000000..439c4c6 --- /dev/null +++ b/tests/tests_cases/10/7c5aebc1feeef4eaf19083019547457b8cf3fc3d/project/project1.txt @@ -0,0 +1 @@ +This is project1 diff --git a/tests/tests_cases/10/c37c77fbf9d12418c83e1510fb0bdece8bd2e11c/README.md b/tests/tests_cases/10/c37c77fbf9d12418c83e1510fb0bdece8bd2e11c/README.md new file mode 100644 index 0000000..b8c1f6a --- /dev/null +++ b/tests/tests_cases/10/c37c77fbf9d12418c83e1510fb0bdece8bd2e11c/README.md @@ -0,0 +1,92 @@ +# Descartes Underwriting + +## Context + +We wish to create a backup tool that will save only the last modified files of a storage unit. + +In our example, the storage unit is **not a bucket**. + +The storage unit is the `DD-MM-YYYY-test` branch of the current `descartes-underwriting/devops-technical-test-data` git repository. + +## Property + +The `descartes-underwriting/devops-technical-test-data` repository is not frozen and will have new commits. + +Commits will be added to the `DD-MM-YYYY-test` branch multiple times every day. + +The `DD-MM-YYYY-test` branch name will be adapted using standard datetime convention eg: `01-01-2022-test` for the 1st of January 2022. + +## Task + +Develop a backup tool to save the modified files at each commit. + +### Submission + +Script and data should be saved on a private `candidate/descartes-backup-project` repository on your github account. + +Access should be granted to all members of the `descartes-underwriting` group: + + + +Especially: + +* +* +* + +### Script + +Create a script to automate the backup process using open source software. + +The script should track the changes fo the branch `DD-MM-YYYY-test` of the `descartes-underwriting/devops-technical-test-data` repository. + +The execution of the script should be carried out with a github-action / gitlab-pipeline or any other tool automating git workflow on your git project. + +It is highly recommended to use a scheduling tool to execute the back up process. + +### Data + +The backup should store files in separate folders. + +The backup file structure should be based on the sha1 of the `descartes-underwriting/devops-technical-test-data`. + +Starting from the initial commit [282180fe7e5d9cbf297f2f0ef813cffe60ce2328](https://github.com/descartes-underwriting/devops-technical-test-data/commit/282180fe7e5d9cbf297f2f0ef813cffe60ce2328), all the history should be backup. + +## File structure example + +For the following commits on the `descartes-underwriting/devops-technical-test-data`: + +| SHA | OPERATION | +|-----|-----------| +| Commit_N | create readme.md | +| Commit_N+1 | create doc.txt | +| Commit_N+2 | create data/test/test.txt | +| Commit_N+3 | append text to ./doc.txt | +| Commit_N+4 | create test/project/project1.txt | + +The `candidate/descartes-backup-project` repository should have + +```bash +$ tree . +. +├── .gitworkflow +│   └── workflows +│   └── my-lovely-workflow.yml +├── data +│   ├── N +│   │   └── readme.md +│   ├── N+1 +│   │   └── doc.txt +│   ├── N+2 +│   │   └── data +│   │   └── test +│   │   └── test.txt +│   ├── N+3 +│   │   └── doc.txt +│   └── N+4 +│   └── test +│   └── project +│   └── project1.txt +└── script + └── my-beautiful-script.best-language +``` diff --git a/tests/tests_cases/2/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md b/tests/tests_cases/2/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md new file mode 100644 index 0000000..9ba167d --- /dev/null +++ b/tests/tests_cases/2/282180fe7e5d9cbf297f2f0ef813cffe60ce2328/README.md @@ -0,0 +1,78 @@ +# Descartes Underwriting + +## Context + +We wish to create a backup tool that will save only the last modified files of a storage unit. + +In our example, the storage unit is **not a bucket**. + +The storage unit is a git repository called `devops-technical-test-data`. + +The repository `devops-technical-test-data` is not frozen and will have new commits. + +Commits will be added to the `datestamp-test` branch on the `devops-technical-test-data` repository. + +## Task + +Develop a backup tool to save the modified files at each commit. + +### Submission + +Script and data should be saved on a private `descartes-backup-project` repository on your github account. + +Access should be granted to all members of the `descartes-underwriting` group: + + + +### Script + +Create a script to automate the backup process using open source software. + +The script should track the branch `datestamp-test` of the repository. + +The execution of the script should be carried with a github-action / gitlab-pipeline or any other git automated workflow. + +### Data + +The backup should store files in separate folders. + +The backup file structure should be based on the sha1 of the `devops-technical-test-data`. + +## File structure example + +For the following commits on the `devops-technical-test-data`: + +| SHA | OPERATION | +|-----|-----------| +| Commit_N | create readme.md | +| Commit_N+1 | create doc.txt | +| Commit_N+2 | create data/test/test.txt | +| Commit_N+3 | append text to ./doc.txt | +| Commit_N+4 | create test/project/project1.txt | + +The `candidate_backup_repository` should have + +```bash +$ tree . +. +├── .gitworkflow +│   └── workflows +│   └── my-lovely-workflow.yml +├── data +│   ├── N +│   │   └── readme.md +│   ├── N+1 +│   │   └── doc.txt +│   ├── N+2 +│   │   └── data +│   │   └── test +│   │   └── test.txt +│   ├── N+3 +│   │   └── doc.txt +│   └── N+4 +│   └── test +│   └── project +│   └── project1.txt +└── script + └── my-beautiful-script.best-language +``` diff --git a/tests/tests_cases/2/46fe26c9dcf2354a0ed3f304ed6818de9606f7b5/doc.txt b/tests/tests_cases/2/46fe26c9dcf2354a0ed3f304ed6818de9606f7b5/doc.txt new file mode 100644 index 0000000..225fcd6 --- /dev/null +++ b/tests/tests_cases/2/46fe26c9dcf2354a0ed3f304ed6818de9606f7b5/doc.txt @@ -0,0 +1 @@ +This is a doc