1.0.0 Basic working version of the script with Readme
This commit is contained in:
Vivek Teega
commit
483d84080d
2
.gitignore
vendored
Normal file
2
.gitignore
vendored
Normal file
@ -0,0 +1,2 @@
|
||||
config.ini
|
||||
**/
|
||||
87
Readme.md
Normal file
87
Readme.md
Normal file
@ -0,0 +1,87 @@
|
||||
|
||||
# Welcome to Autosync git repos!
|
||||
|
||||
Hi! Autosync git repos is a python script which helps you maintain backup of your GitHub repositories in GitLab and Codeberg. With a combination of the script and cronjob to execute it, make sure you never lose your precious code!
|
||||
|
||||
```mermaid
|
||||
|
||||
graph LR
|
||||
|
||||
A[GitHub] -- Pull --> B(Local storage)
|
||||
|
||||
B(Local storage) -- Push --> C[GitLab]
|
||||
|
||||
B(Local storage) -- Push --> D[Codeberg]
|
||||
|
||||
```
|
||||
|
||||
# Setup
|
||||
|
||||
**Very Important** - If you have cloned the Autosync git repo, then you have to delete the .git folder in its root directory using `sudo rm -r .git`
|
||||
|
||||
## 1. Configure config.ini
|
||||
|
||||
Create config.ini file in the base directory and fill the following values.
|
||||
|
||||
```
|
||||
[DEFAULT]
|
||||
|
||||
GIT_USERNAME = <git-username>
|
||||
GIT_ORG_NAME = <organization-group-name>
|
||||
GITLAB_GROUP_ID = <gitlab-group-id>
|
||||
GITLAB_API_TOKEN = <gitlab-personal-access-token>
|
||||
CODEBERG_API_TOKEN = <codeberg-personal-access-token>
|
||||
GITHUB_API_TOKEN = <github-personal-access-token>
|
||||
```
|
||||
|
||||
You will need to create personal access tokens under your account's 'Settings' tab at https://github.com , https://gitlab.com , https://codeberg.org
|
||||
|
||||
## 2. Configure continuous synchronization
|
||||
|
||||
### Supervisord
|
||||
|
||||
Setup supervisord configuration file for the script
|
||||
|
||||
`vim /etc/supervisor/conf.d/autosync-repos.conf`
|
||||
|
||||
with the following content
|
||||
|
||||
```
|
||||
[program:sync-git-repos]
|
||||
|
||||
directory=<path to base directory>
|
||||
command=/usr/bin/python3 <path to base directory>/sync_repos.py
|
||||
user=<user>
|
||||
autostart=false
|
||||
autorestart=false
|
||||
stopasgroup=true
|
||||
killasgroup=true
|
||||
stderr_logfile=/var/log/sync-git-repos/sync-git-repos.err.log
|
||||
stdout_logfile=/var/log/sync-git-repos/sync-git-repos.out.log
|
||||
|
||||
```
|
||||
|
||||
Initialize supervisor settings with
|
||||
|
||||
```
|
||||
sudo mkdir /var/log/sync-git-repos/
|
||||
sudo touch /var/log/sync-git-repos/sync-git-repos.err.log
|
||||
sudo touch /var/log/sync-git-repos/sync-git-repos.out.log
|
||||
|
||||
sudo supervisorctl reread
|
||||
sudo supervisorctl update
|
||||
```
|
||||
|
||||
Start the script `sudo supervisorctl start sync-git-repos`
|
||||
Check script status `sudo supervisorctl status sync-git-repos`
|
||||
|
||||
### Cronjob
|
||||
|
||||
Setup a cronjob to execute the Python script *sync-repos.py*
|
||||
|
||||
```
|
||||
sudo crontab -e
|
||||
|
||||
# Sync every hour of the day
|
||||
0 * * * * supervisorctl start sync-git-repos
|
||||
```
|
||||
7
config-example.ini
Normal file
7
config-example.ini
Normal file
@ -0,0 +1,7 @@
|
||||
[DEFAULT]
|
||||
GIT_USERNAME = <git-username>
|
||||
GIT_ORG_NAME = <organization-group-name>
|
||||
GITLAB_GROUP_ID = <gitlab-group-id>
|
||||
GITLAB_API_TOKEN = <gitlab-personal-access-token>
|
||||
CODEBERG_API_TOKEN = <codeberg-personal-access-token>
|
||||
GITHUB_API_TOKEN = <github-personal-access-token>
|
||||
85
sync_repos.py
Normal file
85
sync_repos.py
Normal file
@ -0,0 +1,85 @@
|
||||
import requests
|
||||
import json
|
||||
import pdb
|
||||
import os
|
||||
from operator import itemgetter
|
||||
import configparser
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
config.read('config.ini')
|
||||
|
||||
GIT_USERNAME = config['DEFAULT']['GIT_USERNAME']
|
||||
GIT_ORG_NAME = config['DEFAULT']['GIT_ORG_NAME']
|
||||
GITLAB_GROUP_ID = config['DEFAULT']['GITLAB_GROUP_ID']
|
||||
GITLAB_API_TOKEN = config['DEFAULT']['GITLAB_API_TOKEN']
|
||||
CODEBERG_API_TOKEN = config['DEFAULT']['CODEBERG_API_TOKEN']
|
||||
GITHUB_API_TOKEN = config['DEFAULT']['GITHUB_API_TOKEN']
|
||||
|
||||
current_working_dir = os.getcwd()
|
||||
|
||||
def repo_cloning():
|
||||
directory_data = {}
|
||||
|
||||
GITHUB_HEADERS = {
|
||||
'Authorization': "token " + GITHUB_API_TOKEN,
|
||||
}
|
||||
response = requests.get(f'https://api.github.com/orgs/{GIT_ORG_NAME}/repos?page=1&per_page=1000&type=all', headers=GITHUB_HEADERS)
|
||||
if response.status_code == 200:
|
||||
response_data = response.json()
|
||||
for i in range(len(response_data)):
|
||||
|
||||
#if response_data[i]['name'] in ['app-guides', "electrumx"]:
|
||||
name, private, visibility, html_url, description = itemgetter('name', 'private', 'visibility', 'html_url', 'description')(response_data[i])
|
||||
directory_data[i] = {
|
||||
'name': name,
|
||||
'private': private,
|
||||
'visibility': visibility,
|
||||
'html_url': html_url,
|
||||
'description': description
|
||||
}
|
||||
|
||||
if not os.path.isdir(f"{current_working_dir}/{directory_data[i]['name']}"):
|
||||
directory_data[i]['available_locally'] = False
|
||||
os.system(f"git clone {directory_data[i]['html_url']}")
|
||||
else:
|
||||
directory_data[i]['available_locally'] = True
|
||||
print('Completed cloning')
|
||||
return directory_data
|
||||
|
||||
def create_repos_gitlab(directory_data):
|
||||
for key in directory_data.keys():
|
||||
name, private, visibility, html_url, description = itemgetter('name', 'private', 'visibility', 'html_url', 'description')(directory_data[key])
|
||||
headers = { 'PRIVATE-TOKEN': GITLAB_API_TOKEN }
|
||||
json_data = {
|
||||
'name': name,
|
||||
'description': description,
|
||||
'path': name,
|
||||
'initialize_with_readme': 'false',
|
||||
'visibility': visibility,
|
||||
'namespace_id': GITLAB_GROUP_ID
|
||||
}
|
||||
response = requests.post('https://gitlab.com/api/v4/projects/', headers=headers, json=json_data)
|
||||
|
||||
def setting_remote_repositories(directory_data):
|
||||
for key in directory_data.keys():
|
||||
if directory_data[key]['available_locally'] == False:
|
||||
repo_name = directory_data[key]['name']
|
||||
print(f'Setting remote repositores for repo_name')
|
||||
os.system(f"cd {current_working_dir}/{repo_name} && git remote set-url origin --add https://codeberg.org/{GIT_ORG_NAME}/{repo_name}.git && git remote set-url origin --add https://gitlab.com/{GIT_ORG_NAME}/{repo_name}.git")
|
||||
|
||||
def pull_push_code(directory_data):
|
||||
for key in directory_data.keys():
|
||||
repo_name = directory_data[key]['name']
|
||||
os.system(f"cd {current_working_dir}/{repo_name} && git pull && git push https://{GITHUB_API_TOKEN}@github.com/{GIT_ORG_NAME}/{repo_name}.git && git push https://{GIT_USERNAME}:{GITLAB_API_TOKEN}@gitlab.com/{GIT_ORG_NAME}/{repo_name}.git")
|
||||
# Codeberg
|
||||
try:
|
||||
os.system(f"cd {current_working_dir}/{repo_name} && git pull && git push https://{CODEBERG_API_TOKEN}@codeberg.org/{GIT_ORG_NAME}/{repo_name}.git")
|
||||
except:
|
||||
print("Couldn't process Codeberg")
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
dir_data = repo_cloning()
|
||||
create_repos_gitlab(dir_data)
|
||||
setting_remote_repositories(dir_data)
|
||||
pull_push_code(dir_data)
|
||||
Loading…
Reference in New Issue
Block a user