1.0.0 Basic working version of the script with Readme
This commit is contained in:
Vivek Teega
commit
483d84080d
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
|||||||
|
config.ini
|
||||||
|
**/
|
||||||
87
Readme.md
Normal file
87
Readme.md
Normal file
@ -0,0 +1,87 @@
|
|||||||
|
|
||||||
|
# Welcome to Autosync git repos!
|
||||||
|
|
||||||
|
Hi! Autosync git repos is a python script which helps you maintain backup of your GitHub repositories in GitLab and Codeberg. With a combination of the script and cronjob to execute it, make sure you never lose your precious code!
|
||||||
|
|
||||||
|
```mermaid
|
||||||
|
|
||||||
|
graph LR
|
||||||
|
|
||||||
|
A[GitHub] -- Pull --> B(Local storage)
|
||||||
|
|
||||||
|
B(Local storage) -- Push --> C[GitLab]
|
||||||
|
|
||||||
|
B(Local storage) -- Push --> D[Codeberg]
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
# Setup
|
||||||
|
|
||||||
|
**Very Important** - If you have cloned the Autosync git repo, then you have to delete the .git folder in its root directory using `sudo rm -r .git`
|
||||||
|
|
||||||
|
## 1. Configure config.ini
|
||||||
|
|
||||||
|
Create config.ini file in the base directory and fill the following values.
|
||||||
|
|
||||||
|
```
|
||||||
|
[DEFAULT]
|
||||||
|
|
||||||
|
GIT_USERNAME = <git-username>
|
||||||
|
GIT_ORG_NAME = <organization-group-name>
|
||||||
|
GITLAB_GROUP_ID = <gitlab-group-id>
|
||||||
|
GITLAB_API_TOKEN = <gitlab-personal-access-token>
|
||||||
|
CODEBERG_API_TOKEN = <codeberg-personal-access-token>
|
||||||
|
GITHUB_API_TOKEN = <github-personal-access-token>
|
||||||
|
```
|
||||||
|
|
||||||
|
You will need to create personal access tokens under your account's 'Settings' tab at https://github.com , https://gitlab.com , https://codeberg.org
|
||||||
|
|
||||||
|
## 2. Configure continuous synchronization
|
||||||
|
|
||||||
|
### Supervisord
|
||||||
|
|
||||||
|
Setup supervisord configuration file for the script
|
||||||
|
|
||||||
|
`vim /etc/supervisor/conf.d/autosync-repos.conf`
|
||||||
|
|
||||||
|
with the following content
|
||||||
|
|
||||||
|
```
|
||||||
|
[program:sync-git-repos]
|
||||||
|
|
||||||
|
directory=<path to base directory>
|
||||||
|
command=/usr/bin/python3 <path to base directory>/sync_repos.py
|
||||||
|
user=<user>
|
||||||
|
autostart=false
|
||||||
|
autorestart=false
|
||||||
|
stopasgroup=true
|
||||||
|
killasgroup=true
|
||||||
|
stderr_logfile=/var/log/sync-git-repos/sync-git-repos.err.log
|
||||||
|
stdout_logfile=/var/log/sync-git-repos/sync-git-repos.out.log
|
||||||
|
|
||||||
|
```
|
||||||
|
|
||||||
|
Initialize supervisor settings with
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo mkdir /var/log/sync-git-repos/
|
||||||
|
sudo touch /var/log/sync-git-repos/sync-git-repos.err.log
|
||||||
|
sudo touch /var/log/sync-git-repos/sync-git-repos.out.log
|
||||||
|
|
||||||
|
sudo supervisorctl reread
|
||||||
|
sudo supervisorctl update
|
||||||
|
```
|
||||||
|
|
||||||
|
Start the script `sudo supervisorctl start sync-git-repos`
|
||||||
|
Check script status `sudo supervisorctl status sync-git-repos`
|
||||||
|
|
||||||
|
### Cronjob
|
||||||
|
|
||||||
|
Setup a cronjob to execute the Python script *sync-repos.py*
|
||||||
|
|
||||||
|
```
|
||||||
|
sudo crontab -e
|
||||||
|
|
||||||
|
# Sync every hour of the day
|
||||||
|
0 * * * * supervisorctl start sync-git-repos
|
||||||
|
```
|
||||||
7
config-example.ini
Normal file
7
config-example.ini
Normal file
@ -0,0 +1,7 @@
|
|||||||
|
[DEFAULT]
|
||||||
|
GIT_USERNAME = <git-username>
|
||||||
|
GIT_ORG_NAME = <organization-group-name>
|
||||||
|
GITLAB_GROUP_ID = <gitlab-group-id>
|
||||||
|
GITLAB_API_TOKEN = <gitlab-personal-access-token>
|
||||||
|
CODEBERG_API_TOKEN = <codeberg-personal-access-token>
|
||||||
|
GITHUB_API_TOKEN = <github-personal-access-token>
|
||||||
85
sync_repos.py
Normal file
85
sync_repos.py
Normal file
@ -0,0 +1,85 @@
|
|||||||
|
import requests
|
||||||
|
import json
|
||||||
|
import pdb
|
||||||
|
import os
|
||||||
|
from operator import itemgetter
|
||||||
|
import configparser
|
||||||
|
|
||||||
|
config = configparser.ConfigParser()
|
||||||
|
config.read('config.ini')
|
||||||
|
|
||||||
|
GIT_USERNAME = config['DEFAULT']['GIT_USERNAME']
|
||||||
|
GIT_ORG_NAME = config['DEFAULT']['GIT_ORG_NAME']
|
||||||
|
GITLAB_GROUP_ID = config['DEFAULT']['GITLAB_GROUP_ID']
|
||||||
|
GITLAB_API_TOKEN = config['DEFAULT']['GITLAB_API_TOKEN']
|
||||||
|
CODEBERG_API_TOKEN = config['DEFAULT']['CODEBERG_API_TOKEN']
|
||||||
|
GITHUB_API_TOKEN = config['DEFAULT']['GITHUB_API_TOKEN']
|
||||||
|
|
||||||
|
current_working_dir = os.getcwd()
|
||||||
|
|
||||||
|
def repo_cloning():
|
||||||
|
directory_data = {}
|
||||||
|
|
||||||
|
GITHUB_HEADERS = {
|
||||||
|
'Authorization': "token " + GITHUB_API_TOKEN,
|
||||||
|
}
|
||||||
|
response = requests.get(f'https://api.github.com/orgs/{GIT_ORG_NAME}/repos?page=1&per_page=1000&type=all', headers=GITHUB_HEADERS)
|
||||||
|
if response.status_code == 200:
|
||||||
|
response_data = response.json()
|
||||||
|
for i in range(len(response_data)):
|
||||||
|
|
||||||
|
#if response_data[i]['name'] in ['app-guides', "electrumx"]:
|
||||||
|
name, private, visibility, html_url, description = itemgetter('name', 'private', 'visibility', 'html_url', 'description')(response_data[i])
|
||||||
|
directory_data[i] = {
|
||||||
|
'name': name,
|
||||||
|
'private': private,
|
||||||
|
'visibility': visibility,
|
||||||
|
'html_url': html_url,
|
||||||
|
'description': description
|
||||||
|
}
|
||||||
|
|
||||||
|
if not os.path.isdir(f"{current_working_dir}/{directory_data[i]['name']}"):
|
||||||
|
directory_data[i]['available_locally'] = False
|
||||||
|
os.system(f"git clone {directory_data[i]['html_url']}")
|
||||||
|
else:
|
||||||
|
directory_data[i]['available_locally'] = True
|
||||||
|
print('Completed cloning')
|
||||||
|
return directory_data
|
||||||
|
|
||||||
|
def create_repos_gitlab(directory_data):
|
||||||
|
for key in directory_data.keys():
|
||||||
|
name, private, visibility, html_url, description = itemgetter('name', 'private', 'visibility', 'html_url', 'description')(directory_data[key])
|
||||||
|
headers = { 'PRIVATE-TOKEN': GITLAB_API_TOKEN }
|
||||||
|
json_data = {
|
||||||
|
'name': name,
|
||||||
|
'description': description,
|
||||||
|
'path': name,
|
||||||
|
'initialize_with_readme': 'false',
|
||||||
|
'visibility': visibility,
|
||||||
|
'namespace_id': GITLAB_GROUP_ID
|
||||||
|
}
|
||||||
|
response = requests.post('https://gitlab.com/api/v4/projects/', headers=headers, json=json_data)
|
||||||
|
|
||||||
|
def setting_remote_repositories(directory_data):
|
||||||
|
for key in directory_data.keys():
|
||||||
|
if directory_data[key]['available_locally'] == False:
|
||||||
|
repo_name = directory_data[key]['name']
|
||||||
|
print(f'Setting remote repositores for repo_name')
|
||||||
|
os.system(f"cd {current_working_dir}/{repo_name} && git remote set-url origin --add https://codeberg.org/{GIT_ORG_NAME}/{repo_name}.git && git remote set-url origin --add https://gitlab.com/{GIT_ORG_NAME}/{repo_name}.git")
|
||||||
|
|
||||||
|
def pull_push_code(directory_data):
|
||||||
|
for key in directory_data.keys():
|
||||||
|
repo_name = directory_data[key]['name']
|
||||||
|
os.system(f"cd {current_working_dir}/{repo_name} && git pull && git push https://{GITHUB_API_TOKEN}@github.com/{GIT_ORG_NAME}/{repo_name}.git && git push https://{GIT_USERNAME}:{GITLAB_API_TOKEN}@gitlab.com/{GIT_ORG_NAME}/{repo_name}.git")
|
||||||
|
# Codeberg
|
||||||
|
try:
|
||||||
|
os.system(f"cd {current_working_dir}/{repo_name} && git pull && git push https://{CODEBERG_API_TOKEN}@codeberg.org/{GIT_ORG_NAME}/{repo_name}.git")
|
||||||
|
except:
|
||||||
|
print("Couldn't process Codeberg")
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
dir_data = repo_cloning()
|
||||||
|
create_repos_gitlab(dir_data)
|
||||||
|
setting_remote_repositories(dir_data)
|
||||||
|
pull_push_code(dir_data)
|
||||||
Loading…
Reference in New Issue
Block a user