From 54c07cf30f82454ae1e1bae77899d5ff1723152f Mon Sep 17 00:00:00 2001 From: asif158 Date: Wed, 3 Jul 2024 10:27:36 +0530 Subject: [PATCH] Add: bootstrap script for fast syncing --- .gitignore | 1 + Dockerfile | 15 ++++--- Dockerfile-testnet | 13 ++++-- README.md | 72 ++++++++++++++++-------------- deb-files-mainnet/supervisord.conf | 33 ++++++++------ deb-files-testnet/supervisord.conf | 23 ++++++++-- start.sh | 47 +++++++++++++++++++ 7 files changed, 144 insertions(+), 60 deletions(-) create mode 100644 .gitignore create mode 100644 start.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2eea525 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.env \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 4ee05c8..8f8cdd8 100755 --- a/Dockerfile +++ b/Dockerfile @@ -3,24 +3,29 @@ FROM ubuntu:22.04 ENV DEBIAN_FRONTEND=noninteractive # Copying files to the container -COPY /deb-files-mainnet/supervisord.conf /tmp COPY /deb-files-mainnet/backend-flo_0.15.1.1-satoshilabs-1_amd64.deb /tmp/backend-flo.deb COPY /deb-files-mainnet/blockbook-flo_0.4.0_amd64.deb /tmp/blockbook-flo.deb # Installing Blockbook and Supervisor RUN apt-get update && \ - apt-get install -y /tmp/backend-flo.deb /tmp/blockbook-flo.deb supervisor && \ + apt-get install -y /tmp/backend-flo.deb /tmp/blockbook-flo.deb supervisor wget && \ rm /tmp/backend-flo.deb /tmp/blockbook-flo.deb && \ rm -rf /var/lib/apt/lists/* +# Copying conf file after installing supervisor to fix scripting error +COPY /deb-files-mainnet/supervisord.conf /etc/supervisor/supervisord.conf +COPY start.sh /tmp + # Configuring backend as forground process under supervisord -RUN sed -i "s/daemon=1/daemon=0/g" /opt/coins/nodes/flo/flo.conf - +RUN sed -i "s/daemon=1/daemon=0/g" /opt/coins/nodes/flo/flo.conf &&\ + chmod +x /tmp/start.sh + EXPOSE 9166 VOLUME /opt/coins -CMD ["/usr/bin/supervisord", "-c", "/tmp/supervisord.conf"] +# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] +CMD ["/bin/bash", "-c", "/tmp/start.sh"] # check logs # tail -f /opt/coins/data/flo/backend/debug.log diff --git a/Dockerfile-testnet b/Dockerfile-testnet index c11dc82..beab447 100644 --- a/Dockerfile-testnet +++ b/Dockerfile-testnet @@ -4,19 +4,23 @@ ENV DEBIAN_FRONTEND=noninteractive # Copying files to the container # COPY /deb-files-testnet/peers.dat /tmp -COPY /deb-files-testnet/supervisord.conf /tmp COPY /deb-files-testnet/backend-flo-testnet_0.15.1.1-satoshilabs-1_amd64.deb /tmp/backend-flo.deb COPY /deb-files-testnet/blockbook-flo-testnet_0.4.0_amd64.deb /tmp/blockbook-flo.deb # Installing Blockbook and Supervisor RUN apt-get update && \ - apt-get install -y /tmp/backend-flo.deb /tmp/blockbook-flo.deb supervisor && \ + apt-get install -y /tmp/backend-flo.deb /tmp/blockbook-flo.deb supervisor wget && \ rm /tmp/backend-flo.deb /tmp/blockbook-flo.deb && \ rm -rf /var/lib/apt/lists/* +# Copying conf file after installing supervisor to fix scripting error +COPY /deb-files-testnet/supervisord.conf /etc/supervisor/supervisord.conf +COPY start.sh /tmp + # Configuring backend as forground process and adding nodes to sync backend RUN sed -i "s/daemon=1/daemon=0/g" /opt/coins/nodes/flo_testnet/flo_testnet.conf && \ - sed -i '$a addnode=ramanujam.ranchimall.net\naddnode=turing.ranchimall.net\naddnode=stevejobs.ranchimall.net\naddnode=brahmagupta.ranchimall.net\naddnode=feynman.ranchimall.net' /opt/coins/nodes/flo_testnet/flo_testnet.conf + sed -i '$a addnode=ramanujam.ranchimall.net\naddnode=turing.ranchimall.net\naddnode=stevejobs.ranchimall.net\naddnode=brahmagupta.ranchimall.net\naddnode=feynman.ranchimall.net' /opt/coins/nodes/flo_testnet/flo_testnet.conf && \ + chmod +x /tmp/start.sh # RUN cd /opt/coins/nodes/flo_testnet && \ # /opt/coins/nodes/flo_testnet/bin/flod --testnet -datadir=/opt/coins/data/flo_testnet/backend -conf=/opt/coins/nodes/flo_testnet/flo_testnet.conf -pid=/run/flo_testnet/flo_testnet.pid @@ -27,7 +31,8 @@ EXPOSE 19166 VOLUME /opt/coins -CMD ["/usr/bin/supervisord", "-c", "/tmp/supervisord.conf"] +# CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/supervisord.conf"] +CMD ["/bin/bash", "-c", "/tmp/start.sh"] # check logs # tail -f /opt/coins/data/flo_testnet/backend/testnet4/debug.log diff --git a/README.md b/README.md index c95c41b..677171d 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ This guide will help you build and run the Docker image for Blockbook Mainnet an ## Prerequisites -- [Docker installed on your machine](https://docs.docker.com/engine/install/). -- Clone or download the repository to your local machine. +- [Docker installed on your machine](https://docs.docker.com/engine/install/). +- Clone or download the repository to your local machine. --- @@ -13,7 +13,7 @@ This guide will help you build and run the Docker image for Blockbook Mainnet an Steps to build the Docker image for Mainnet. -- **Building the Docker Image** +- **Building the Docker Image** Navigate to the directory where the repository is cloned and build the Docker image: @@ -23,7 +23,7 @@ Steps to build the Docker image for Mainnet. docker build -f Dockerfile -t ranchimall/blockbook-mainnet:1.0.0 . ``` -- **Running the Docker Container** +- **Running the Docker Container** Create a named volume for persistent storage: @@ -34,20 +34,14 @@ Steps to build the Docker image for Mainnet. Run the Docker container in detached mode and map port `9166` of the container to port `9166` (or any other port) on the host machine (`-p host-port:container-port`): ```sh - docker run -d --name blockbook-mainnet -p 9166:9166 --mount source=mainnet,target=/opt/coins ranchimall/blockbook-mainnet:1.0.0 + docker run -d -e ZIP=false -e NETWORK=mainnet --name blockbook-mainnet -p 9166:9166 --mount source=mainnet,target=/opt/coins ranchimall/blockbook-mainnet:1.0.0 ``` -- **Accessing the Frontend** +- **Accessing the Frontend** - Once the Docker container is running, you can access the frontend of the application by navigating to: + Once the Docker container is running, you can access the frontend of the application by navigating to: [https://localhost:9166](https://localhost:9166) in your web browser. - ```sh - https://localhost:9166 - ``` - - in your web browser. - -- **Accessing Logs** +- **Accessing Logs** To check the logs, you can access the container shell and use the `tail` command: @@ -71,7 +65,7 @@ Steps to build the Docker image for Mainnet. Steps to build the Docker image for Testnet. -- **Building the Docker Image** +- **Building the Docker Image** Navigate to the directory where the repository is cloned or downloaded and build the Docker image: @@ -80,7 +74,7 @@ Steps to build the Docker image for Testnet. docker build -f Dockerfile-testnet -t ranchimall/blockbook-testnet:1.0.0 . ``` -- **Running the Docker Container** +- **Running the Docker Container** Create a named volume for persistent storage: @@ -91,20 +85,14 @@ Steps to build the Docker image for Testnet. Run the Docker container in detached mode and map port `19166` of the container to port `19166` (or any other port) on the host machine (`-p host-port:container-port`): ```sh - docker run -d --name blockbook-testnet -p 19166:19166 --mount source=testnet,target=/opt/coins ranchimall/blockbook-testnet:1.0.0 + docker run -d -e ZIP=false -e NETWORK=testnet --name blockbook-testnet -p 19166:19166 --mount source=testnet,target=/opt/coins ranchimall/blockbook-testnet:1.0.0 ``` -- **Accessing the Frontend** +- **Accessing the Frontend** - Once the Docker container is running, you can access the frontend of the application by navigating to: + Once the Docker container is running, you can access the frontend of the application by navigating to: [https://localhost:19166](https://localhost:19166) in your web browser. - ```sh - https://localhost:19166 - ``` - - in your web browser. - -- **Accessing Logs** +- **Accessing Logs** To check the logs, you can access the container shell and use the `tail` command: @@ -122,9 +110,27 @@ Steps to build the Docker image for Testnet. tail -f /opt/coins/blockbook/flo_testnet/logs/blockbook.INFO ``` +## Environment Variables + +Blockbook uses environment variables to allow for configuration settings. You set the environment variables in your Docker run startup command. Here are the config settings offered by this image: + +- NETWORK: [mainnet|testnet] The FLO network you wish to run the Blockbook on. +- ZIP: [true|false] To download the bootstrap data or not to speed up the syncing of blocks, which will be reflected in Blockbook. +- TESTNET_ZIP_URL: The URL for Testnet to download the zip for bootstrapping Blockbook (Testnet). +- MAINNET_ZIP_URL: The URL for Mainnet to download the zip for bootstrapping Blockbook (Mainnet). + +You can use the `--env-file` flag to pass these environments through a file during runtime. Create a `.env` file in the same directory as your project, using the `.env.example` file as a guide. Then, run your app with the following command to load the environment variables from the file: +```sh +#to run mainnet add mainnet Environment Variables. +docker run -d --env-file .env --name blockbook-mainnet -p 9166:9166 --mount source=mainnet,target=/opt/coins ranchimall/blockbook-mainnet:1.0.0 + +#to run testnet add testnet Environment Variables. +docker run -d --env-file .env -name blockbook-testnet -p 19166:19166 --mount source=testnet,target=/opt/coins ranchimall/blockbook-testnet:1.0.0 +``` + ## Additional Commands -- **Stopping the Docker Container:** +- **Stopping the Docker Container:** To stop the container, use: @@ -132,7 +138,7 @@ Steps to build the Docker image for Testnet. docker stop ``` -- **Removing the Docker Container:** +- **Removing the Docker Container:** To remove the container, use: @@ -140,7 +146,7 @@ Steps to build the Docker image for Testnet. docker rm ``` -- **Removing the Docker Image:** +- **Removing the Docker Image:** To remove the image, use: @@ -152,11 +158,9 @@ Replace `` and `` with the actual image name and conta ## Troubleshooting -- Ensure that no other application is using port `9166` or `19166` on your host machine. -- If you encounter issues, check the Docker container logs: +- Ensure that no other application is using port `9166` or `19166` on your host machine. +- If you encounter issues, check the Docker container logs: ```sh docker logs - ``` - ---- + ``` \ No newline at end of file diff --git a/deb-files-mainnet/supervisord.conf b/deb-files-mainnet/supervisord.conf index 416a0d6..1f394bf 100644 --- a/deb-files-mainnet/supervisord.conf +++ b/deb-files-mainnet/supervisord.conf @@ -1,30 +1,37 @@ [supervisord] nodaemon=true +logfile=/var/log/supervisor/supervisord.log +pidfile=/var/run/supervisord.pid +childlogdir=/var/log/supervisor + +# Enable communication to supervised processes +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0700 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock [program:backend-flo] command=/opt/coins/nodes/flo/bin/flod -datadir=/opt/coins/data/flo/backend -conf=/opt/coins/nodes/flo/flo.conf -pid=/run/flo/flo.pid directory=/opt/coins/nodes/flo -user=flo autostart=true autorestart=true -startretries=3 -stderr_logfile=/var/log/supervisor/backend-flo.err.log +startsecs=10 stdout_logfile=/var/log/supervisor/backend-flo.out.log -stopsignal=TERM -stopwaitsecs=300 +stderr_logfile=/var/log/supervisor/backend-flo.err.log [program:blockbook-flo] command=/opt/coins/blockbook/flo/bin/blockbook -blockchaincfg=/opt/coins/blockbook/flo/config/blockchaincfg.json -datadir=/opt/coins/data/flo/blockbook/db -sync -internal=:9066 -public=:9166 -certfile=/opt/coins/blockbook/flo/cert/blockbook -explorer= -log_dir=/opt/coins/blockbook/flo/logs -dbcache=1073741824 directory=/opt/coins/blockbook/flo -user=blockbook-flo autostart=true autorestart=true -startretries=3 -stderr_logfile=/var/log/supervisor/blockbook-flo.err.log +startsecs=10 stdout_logfile=/var/log/supervisor/blockbook-flo.out.log -stopsignal=TERM -stopwaitsecs=300 +stderr_logfile=/var/log/supervisor/blockbook-flo.err.log -[group:coins] -programs=backend-flo,blockbook-flo -priority=999 +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/deb-files-testnet/supervisord.conf b/deb-files-testnet/supervisord.conf index 2bc1b46..508e39c 100644 --- a/deb-files-testnet/supervisord.conf +++ b/deb-files-testnet/supervisord.conf @@ -1,10 +1,23 @@ [supervisord] nodaemon=true +logfile=/var/log/supervisor/supervisord.log +pidfile=/var/run/supervisord.pid +childlogdir=/var/log/supervisor + +# Enable communication to supervised processes +[unix_http_server] +file=/var/run/supervisor.sock +chmod=0700 + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock [program:backend-flo-testnet] -directory=/opt/coins/nodes/flo_testnet command=/opt/coins/nodes/flo_testnet/bin/flod --testnet -datadir=/opt/coins/data/flo_testnet/backend -conf=/opt/coins/nodes/flo_testnet/flo_testnet.conf -pid=/run/flo_testnet/flo_testnet.pid -user=flo +directory=/opt/coins/nodes/flo_testnet autostart=true autorestart=true startsecs=10 @@ -12,11 +25,13 @@ stdout_logfile=/var/log/supervisor/backend-flo-testnet.log stderr_logfile=/var/log/supervisor/backend-flo-testnet.err [program:blockbook-flo-testnet] -directory=/opt/coins/blockbook/flo_testnet command=/opt/coins/blockbook/flo_testnet/bin/blockbook -blockchaincfg=/opt/coins/blockbook/flo_testnet/config/blockchaincfg.json -datadir=/opt/coins/data/flo_testnet/blockbook/db -sync -internal=:19066 -public=:19166 -certfile=/opt/coins/blockbook/flo_testnet/cert/blockbook -explorer= -log_dir=/opt/coins/blockbook/flo_testnet/logs -dbcache=1073741824 -user=blockbook-flo +directory=/opt/coins/blockbook/flo_testnet autostart=true autorestart=true startsecs=10 stdout_logfile=/var/log/supervisor/blockbook-flo-testnet.log stderr_logfile=/var/log/supervisor/blockbook-flo-testnet.err + +[include] +files = /etc/supervisor/conf.d/*.conf diff --git a/start.sh b/start.sh new file mode 100644 index 0000000..c9a4102 --- /dev/null +++ b/start.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +DOWNLOAD_DIR="/tmp" +EXTRACT_DIR="/opt/coins" +ZIP_FILE="${DOWNLOAD_DIR}/data.tar.gz" +SUPERVISOR_CONF="/etc/supervisor/supervisord.conf" + +mkdir -p ${DOWNLOAD_DIR} + +start_supervisor() { + echo "Starting Supervisor..." + /usr/bin/supervisord -c ${SUPERVISOR_CONF} +} +#setting the zip url to mainnet or testnet (bootstrap) +if [ "${NETWORK}" = "testnet" ]; then + ZIP_URL="${TESTNET_ZIP_URL}" +elif [ "${NETWORK}" = "mainnet" ]; then + ZIP_URL="${MAINNET_ZIP_URL}" +else + echo "Invalid or missing NETWORK variable. Please set NETWORK to 'testnet' or 'mainnet'." + exit 1 +fi + +#to download bootstrap or not. +if [ "${ZIP}" = "true" ]; then + echo "Downloading ${ZIP_URL}..." + wget -q --show-progress ${ZIP_URL} -O ${ZIP_FILE} + + if [ $? -ne 0 ]; then + echo "Failed to download the zip file." + exit 1 + fi + + echo "Extracting ${ZIP_FILE} to ${EXTRACT_DIR}..." + tar -xzf ${ZIP_FILE} -C ${EXTRACT_DIR} + + if [ $? -ne 0 ]; then + echo "Failed to extract the zip file." + exit 1 + fi + + rm ${ZIP_FILE} +else + echo "Skipping download and extraction of the zip file." +fi + +start_supervisor