commit 483d84080d33e2cbf4d41484935beb6e9dcbee91 Author: Vivek Teega Date: Mon Jul 4 01:48:27 2022 +0530 1.0.0 Basic working version of the script with Readme diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7fb6b0c --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +config.ini +**/ \ No newline at end of file diff --git a/Readme.md b/Readme.md new file mode 100644 index 0000000..274f702 --- /dev/null +++ b/Readme.md @@ -0,0 +1,87 @@ + +# Welcome to Autosync git repos! + +Hi! Autosync git repos is a python script which helps you maintain backup of your GitHub repositories in GitLab and Codeberg. With a combination of the script and cronjob to execute it, make sure you never lose your precious code! + +```mermaid + +graph LR + +A[GitHub] -- Pull --> B(Local storage) + +B(Local storage) -- Push --> C[GitLab] + +B(Local storage) -- Push --> D[Codeberg] + +``` + +# Setup + +**Very Important** - If you have cloned the Autosync git repo, then you have to delete the .git folder in its root directory using `sudo rm -r .git` + +## 1. Configure config.ini + +Create config.ini file in the base directory and fill the following values. + +``` +[DEFAULT] + +GIT_USERNAME = +GIT_ORG_NAME = +GITLAB_GROUP_ID = +GITLAB_API_TOKEN = +CODEBERG_API_TOKEN = +GITHUB_API_TOKEN = +``` + +You will need to create personal access tokens under your account's 'Settings' tab at https://github.com , https://gitlab.com , https://codeberg.org + +## 2. Configure continuous synchronization + +### Supervisord + +Setup supervisord configuration file for the script + +`vim /etc/supervisor/conf.d/autosync-repos.conf` + +with the following content + +``` +[program:sync-git-repos] + +directory= +command=/usr/bin/python3 /sync_repos.py +user= +autostart=false +autorestart=false +stopasgroup=true +killasgroup=true +stderr_logfile=/var/log/sync-git-repos/sync-git-repos.err.log +stdout_logfile=/var/log/sync-git-repos/sync-git-repos.out.log + +``` + +Initialize supervisor settings with + +``` +sudo mkdir /var/log/sync-git-repos/ +sudo touch /var/log/sync-git-repos/sync-git-repos.err.log +sudo touch /var/log/sync-git-repos/sync-git-repos.out.log + +sudo supervisorctl reread +sudo supervisorctl update +``` + +Start the script `sudo supervisorctl start sync-git-repos` +Check script status `sudo supervisorctl status sync-git-repos` + +### Cronjob + +Setup a cronjob to execute the Python script *sync-repos.py* + +``` +sudo crontab -e + +# Sync every hour of the day +0 * * * * supervisorctl start sync-git-repos +``` diff --git a/config-example.ini b/config-example.ini new file mode 100644 index 0000000..0bf32af --- /dev/null +++ b/config-example.ini @@ -0,0 +1,7 @@ +[DEFAULT] +GIT_USERNAME = +GIT_ORG_NAME = +GITLAB_GROUP_ID = +GITLAB_API_TOKEN = +CODEBERG_API_TOKEN = +GITHUB_API_TOKEN = \ No newline at end of file diff --git a/sync_repos.py b/sync_repos.py new file mode 100644 index 0000000..562336a --- /dev/null +++ b/sync_repos.py @@ -0,0 +1,85 @@ +import requests +import json +import pdb +import os +from operator import itemgetter +import configparser + +config = configparser.ConfigParser() +config.read('config.ini') + +GIT_USERNAME = config['DEFAULT']['GIT_USERNAME'] +GIT_ORG_NAME = config['DEFAULT']['GIT_ORG_NAME'] +GITLAB_GROUP_ID = config['DEFAULT']['GITLAB_GROUP_ID'] +GITLAB_API_TOKEN = config['DEFAULT']['GITLAB_API_TOKEN'] +CODEBERG_API_TOKEN = config['DEFAULT']['CODEBERG_API_TOKEN'] +GITHUB_API_TOKEN = config['DEFAULT']['GITHUB_API_TOKEN'] + +current_working_dir = os.getcwd() + +def repo_cloning(): + directory_data = {} + + GITHUB_HEADERS = { + 'Authorization': "token " + GITHUB_API_TOKEN, + } + response = requests.get(f'https://api.github.com/orgs/{GIT_ORG_NAME}/repos?page=1&per_page=1000&type=all', headers=GITHUB_HEADERS) + if response.status_code == 200: + response_data = response.json() + for i in range(len(response_data)): + + #if response_data[i]['name'] in ['app-guides', "electrumx"]: + name, private, visibility, html_url, description = itemgetter('name', 'private', 'visibility', 'html_url', 'description')(response_data[i]) + directory_data[i] = { + 'name': name, + 'private': private, + 'visibility': visibility, + 'html_url': html_url, + 'description': description + } + + if not os.path.isdir(f"{current_working_dir}/{directory_data[i]['name']}"): + directory_data[i]['available_locally'] = False + os.system(f"git clone {directory_data[i]['html_url']}") + else: + directory_data[i]['available_locally'] = True + print('Completed cloning') + return directory_data + +def create_repos_gitlab(directory_data): + for key in directory_data.keys(): + name, private, visibility, html_url, description = itemgetter('name', 'private', 'visibility', 'html_url', 'description')(directory_data[key]) + headers = { 'PRIVATE-TOKEN': GITLAB_API_TOKEN } + json_data = { + 'name': name, + 'description': description, + 'path': name, + 'initialize_with_readme': 'false', + 'visibility': visibility, + 'namespace_id': GITLAB_GROUP_ID + } + response = requests.post('https://gitlab.com/api/v4/projects/', headers=headers, json=json_data) + +def setting_remote_repositories(directory_data): + for key in directory_data.keys(): + if directory_data[key]['available_locally'] == False: + repo_name = directory_data[key]['name'] + print(f'Setting remote repositores for repo_name') + os.system(f"cd {current_working_dir}/{repo_name} && git remote set-url origin --add https://codeberg.org/{GIT_ORG_NAME}/{repo_name}.git && git remote set-url origin --add https://gitlab.com/{GIT_ORG_NAME}/{repo_name}.git") + +def pull_push_code(directory_data): + for key in directory_data.keys(): + repo_name = directory_data[key]['name'] + os.system(f"cd {current_working_dir}/{repo_name} && git pull && git push https://{GITHUB_API_TOKEN}@github.com/{GIT_ORG_NAME}/{repo_name}.git && git push https://{GIT_USERNAME}:{GITLAB_API_TOKEN}@gitlab.com/{GIT_ORG_NAME}/{repo_name}.git") + # Codeberg + try: + os.system(f"cd {current_working_dir}/{repo_name} && git pull && git push https://{CODEBERG_API_TOKEN}@codeberg.org/{GIT_ORG_NAME}/{repo_name}.git") + except: + print("Couldn't process Codeberg") + + +if __name__ == '__main__': + dir_data = repo_cloning() + create_repos_gitlab(dir_data) + setting_remote_repositories(dir_data) + pull_push_code(dir_data) \ No newline at end of file