From 1d2eacc849c6875e57833f3cc35f74274bfb5ba7 Mon Sep 17 00:00:00 2001 From: lhie1 Date: Sun, 22 Nov 2020 14:47:10 +0800 Subject: [PATCH] Auto Sync --- .github/workflows/Sync.yml | 19 +++ Dockerfile | 8 ++ action.yml | 62 ++++++++++ entrypoint.sh | 248 +++++++++++++++++++++++++++++++++++++ 4 files changed, 337 insertions(+) create mode 100755 .github/workflows/Sync.yml create mode 100644 Dockerfile create mode 100644 action.yml create mode 100755 entrypoint.sh diff --git a/.github/workflows/Sync.yml b/.github/workflows/Sync.yml new file mode 100755 index 0000000..fc6980f --- /dev/null +++ b/.github/workflows/Sync.yml @@ -0,0 +1,19 @@ +on: push +name: Auto Sync +jobs: + run: + name: Run + runs-on: ubuntu-latest + steps: + - name: Checkout source codes + uses: actions/checkout@v1 + - name: Mirror Github to Gitee with white list + uses: lhie1/Rules@master + with: + src: github/lhie1 + dst: gitee/lhie1 + dst_key: ${{ secrets.GITEE_KEY }} + dst_token: ${{ secrets.GITEE_TOKEN }} + white_list: 'Rules' + force_update: true + debug: false \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c5f6f0c --- /dev/null +++ b/Dockerfile @@ -0,0 +1,8 @@ +FROM alpine + +RUN apk add --no-cache git openssh-client bash jq curl&& \ + echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config + +ADD *.sh / + +ENTRYPOINT ["/entrypoint.sh"] \ No newline at end of file diff --git a/action.yml b/action.yml new file mode 100644 index 0000000..4db9575 --- /dev/null +++ b/action.yml @@ -0,0 +1,62 @@ +name: "Hub Mirror Action." +description: "Mirror the organization repos between hub (github/gitee)." +author: "yikun" +branding: + icon: "upload-cloud" + color: "gray-dark" +inputs: + dst_key: + description: "The private SSH key which is used to to push code in destination hub." + required: true + dst_token: + description: "The app token which is used to create repo in destination hub." + required: true + dst: + description: "Destination name. Such as `gitee/kunpengcompute`." + required: true + src: + description: "Source name. Such as `github/kunpengcompute`." + required: true + account_type: + description: "The account type. Such as org, user." + default: 'user' + clone_style: + description: "The git clone style, https or ssh." + default: 'https' + cache_path: + description: "The path to cache the source repos code." + default: '/github/workspace/hub-mirror-cache' + black_list: + description: "Hight priority, the back list of mirror repo. like 'repo1,repo2,repo3'" + default: '' + white_list: + description: "Low priority, the white list of mirror repo. like 'repo1,repo2,repo3'" + default: '' + static_list: + description: "Only mirror repo in the static list, but don't get from repo api (the white/black list is still available). like 'repo1,repo2,repo3'" + default: '' + force_update: + description: "Force to update the destination repo, use '-f' flag do 'git push'" + default: false + debug: + description: "Enable the debug flag to show detail log" + default: false + timeout: + description: "Set the timeout for every git command, like '600'=>600s, '30m'=>30 mins, '1h'=>1 hours" + default: '30m' +runs: + using: "docker" + image: "Dockerfile" + args: + - ${{ inputs.dst_key }} + - ${{ inputs.dst_token }} + - ${{ inputs.src }} + - ${{ inputs.dst }} + - ${{ inputs.account_type }} + - ${{ inputs.clone_style }} + - ${{ inputs.cache_path }} + - ${{ inputs.black_list }} + - ${{ inputs.white_list }} + - ${{ inputs.static_list }} + - ${{ inputs.force_update}} + - ${{ inputs.debug}} diff --git a/entrypoint.sh b/entrypoint.sh new file mode 100755 index 0000000..8806188 --- /dev/null +++ b/entrypoint.sh @@ -0,0 +1,248 @@ +#!/bin/bash + +DEBUG="${INPUT_DEBUG}" + +if [[ "$DEBUG" == "true" ]]; then + set -x +fi + +mkdir -p /root/.ssh +echo "${INPUT_DST_KEY}" > /root/.ssh/id_rsa +chmod 600 /root/.ssh/id_rsa + +DST_TOKEN="${INPUT_DST_TOKEN}" + +SRC_HUB="${INPUT_SRC}" +DST_HUB="${INPUT_DST}" + +ACCOUNT_TYPE="${INPUT_ACCOUNT_TYPE}" + +SRC_TYPE=`dirname $SRC_HUB` +DST_TYPE=`dirname $DST_HUB` + +SRC_ACCOUNT=`basename $SRC_HUB` +DST_ACCOUNT=`basename $DST_HUB` + +CLONE_STYLE="${INPUT_CLONE_STYLE}" + +CACHE_PATH="${INPUT_CACHE_PATH}" + +WHITE_LIST="${INPUT_WHITE_LIST}" +BLACK_LIST="${INPUT_BLACK_LIST}" +STATIC_LIST="${INPUT_STATIC_LIST}" + +FORCE_UPDATE="${INPUT_FORCE_UPDATE}" + +DELAY_EXIT=false + +TIME_OUT="${INPUT_TIMEOUT}" +RETRY_TIMES=3 + +function err_exit { + echo -e "\033[31m $1 \033[0m" + exit 1 +} + +FAILED_LIST=() + +function delay_exit { + echo -e "\033[31m $1 \033[0m" + FAILED_LIST+=($2) + DELAY_EXIT=true + return 1 +} + +if [[ "$ACCOUNT_TYPE" == "org" ]]; then + SRC_LIST_URL_SUFFIX=orgs/$SRC_ACCOUNT/repos + DST_LIST_URL_SUFFIX=orgs/$DST_ACCOUNT/repos + DST_CREATE_URL_SUFFIX=orgs/$DST_ACCOUNT/repos +elif [[ "$ACCOUNT_TYPE" == "user" ]]; then + SRC_LIST_URL_SUFFIX=users/$SRC_ACCOUNT/repos + DST_LIST_URL_SUFFIX=users/$DST_ACCOUNT/repos + DST_CREATE_URL_SUFFIX=user/repos +else + err_exit "Unknown account type, the `account_type` should be `user` or `org`" +fi + +if [[ "$SRC_TYPE" == "github" ]]; then + SRC_REPO_LIST_API=https://api.github.com/$SRC_LIST_URL_SUFFIX + if [[ "$CLONE_STYLE" == "ssh" ]]; then + SRC_REPO_BASE_URL=git@github.com: + elif [[ "$CLONE_STYLE" == "https" ]]; then + SRC_REPO_BASE_URL=https://github.com/ + fi +elif [[ "$SRC_TYPE" == "gitee" ]]; then + SRC_REPO_LIST_API=https://gitee.com/api/v5/$SRC_LIST_URL_SUFFIX + if [[ "$CLONE_STYLE" == "ssh" ]]; then + SRC_REPO_BASE_URL=git@gitee.com: + elif [[ "$CLONE_STYLE" == "https" ]]; then + SRC_REPO_BASE_URL=https://gitee.com/ + fi +else + err_exit "Unknown src args, the `src` should be `[github|gittee]/account`" +fi + +function retry { + local retries=$RETRY_TIMES + local count=0 + until timeout $TIME_OUT "$@"; do + exit=$? + wait=$((2 ** $count)) + count=$(($count + 1)) + if [ $count -lt $retries ]; then + echo "Retry $count/$retries exited $exit, retrying in $wait seconds..." + sleep $wait + else + echo "Retry $count/$retries exited $exit, no more retries left." + return $exit + fi + done + return 0 +} + +function get_all_repo_names +{ + PAGE_NUM=100 + URL=$1 + HUB_TYPE=$2 + if [[ "$HUB_TYPE" == "github" ]]; then + total=`curl -sI "$URL?page=1&per_page=$PAGE_NUM" | sed -nr "s/^[lL]ink:.*page=([0-9]+)&per_page=$PAGE_NUM.*/\1/p"` + elif [[ "$HUB_TYPE" == "gitee" ]]; then + total=`curl -sI "$URL?page=1&per_page=$PAGE_NUM" | grep total_page: |cut -d ' ' -f2 |tr -d '\r'` + fi + + # use pagination? + if [ -z "$total" ]; then + # no - this result has only one page + total=1 + fi + + p=1 + while [ "$p" -le "$total" ]; do + x=`curl -s "$URL?page=$p&per_page=$PAGE_NUM" | jq '.[] | .name' | sed 's/"//g'` + echo $x + p=$(($p + 1)) + done +} + +if [[ -z $STATIC_LIST ]]; then + SRC_REPOS=`get_all_repo_names $SRC_REPO_LIST_API $SRC_TYPE` +else + SRC_REPOS=`echo $STATIC_LIST | tr ',' ' '` +fi + +if [[ "$DST_TYPE" == "github" ]]; then + DST_REPO_CREATE_API=https://api.github.com/$DST_CREATE_URL_SUFFIX + DST_REPO_LIST_API=https://api.github.com/$DST_LIST_URL_SUFFIX +elif [[ "$DST_TYPE" == "gitee" ]]; then + DST_REPO_CREATE_API=https://gitee.com/api/v5/$DST_CREATE_URL_SUFFIX + DST_REPO_LIST_API=https://gitee.com/api/v5/$DST_LIST_URL_SUFFIX +else + err_exit "Unknown dst args, the `dst` should be `[github|gittee]/account`" +fi + +DST_REPOS=`get_all_repo_names $DST_REPO_LIST_API $DST_TYPE` + +function clone_repo +{ + echo -e "\033[31m(0/3)\033[0m" "Downloading..." + if [ ! -d "$1" ]; then + retry git clone $SRC_REPO_BASE_URL$SRC_ACCOUNT/$1.git + fi + cd $1 +} + +function create_repo +{ + # Auto create non-existing repo + has_repo=`echo $DST_REPOS | grep -w $1 | wc -l` + if [ $has_repo == 0 ]; then + echo "Create non-exist repo..." + if [[ "$DST_TYPE" == "github" ]]; then + curl -s -H "Authorization: token $2" --data '{"name":"'$1'"}' $DST_REPO_CREATE_API > /dev/null + elif [[ "$DST_TYPE" == "gitee" ]]; then + curl -s -X POST --header 'Content-Type: application/json;charset=UTF-8' $DST_REPO_CREATE_API -d '{"name": "'$1'","access_token": "'$2'"}' > /dev/null + fi + fi + git remote add $DST_TYPE git@$DST_TYPE.com:$DST_ACCOUNT/$1.git || echo "Remote already exists." +} + +function update_repo +{ + echo -e "\033[31m(1/3)\033[0m" "Updating..." + retry git pull -p +} + +function import_repo +{ + echo -e "\033[31m(2/3)\033[0m" "Importing..." + git remote set-head origin -d + if [[ "$FORCE_UPDATE" == "true" ]]; then + retry git push -f $DST_TYPE refs/remotes/origin/*:refs/heads/* --tags --prune + else + retry git push $DST_TYPE refs/remotes/origin/*:refs/heads/* --tags --prune + fi +} + +function _check_in_list () { + local e match="$1" + shift + for e; do [[ "$e" == "$match" ]] && return 0; done + return 1 +} + +function test_black_white_list +{ + WHITE_ARR=(`echo $WHITE_LIST | tr ',' ' '`) + BLACK_ARR=(`echo $BLACK_LIST | tr ',' ' '`) + _check_in_list $1 "${WHITE_ARR[@]}";in_white_list=$? + _check_in_list $1 "${BLACK_ARR[@]}";in_back_list=$? + + if [[ $in_back_list -ne 0 ]] ; then + if [[ -z $WHITE_LIST ]] || [[ $in_white_list -eq 0 ]] ; then + return 0 + else + echo "Skip, "$1" not in non-empty white list"$WHITE_LIST + return 1 + fi + else + echo "Skip, "$1 "in black list: "$BLACK_LIST + return 1 + fi +} + +if [ ! -d "$CACHE_PATH" ]; then + mkdir -p $CACHE_PATH +fi +cd $CACHE_PATH + +all=0 +success=0 +skip=0 +for repo in $SRC_REPOS +{ + all=$(($all + 1)) + if test_black_white_list $repo ; then + echo -e "\n\033[31mBackup $repo ...\033[0m" + + cd $CACHE_PATH + + clone_repo $repo || delay_exit "clone and cd failed" $repo || continue + + create_repo $repo $DST_TOKEN || delay_exit "create failed" $repo || continue + + update_repo || delay_exit "Update failed" $repo || continue + + import_repo && success=$(($success + 1)) || delay_exit "Push failed" $repo || continue + else + skip=$(($skip + 1)) + fi +} + +failed=$(($all - $skip - $success)) +echo "Total: $all, skip: $skip, successed: $success, failed: $failed." +echo "Failed: "$FAILED_LIST + +if [[ "$DELAY_EXIT" == "true" ]]; then + exit 1 +fi