Please visit our official website at https://smart-scripts.org/.
Linux 5.4.0-1038-aws #40~18.04.1-Ubuntu SMP Sat Feb 6 01:56:56 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux
- Setup basic environment
sudo apt update
sudo apt upgrade
sudo apt install build-essential
In Security from AWS console, allow all TCP connections.
- Ports:
80 SmartScript main 443 SmartScript https 1443 adminMongo
5000 CLCheck server 8080 CLCheck web
- A working example in AWS
Inbound rules (6)
Type | Protocol | Port range | Source |
---|---|---|---|
HTTP | TCP | 80 | 0.0.0.0/0 |
Custom TCP | TCP | 8080 | 0.0.0.0/0 |
SSH | TCP | 22 | 0.0.0.0/0 |
Custom TCP | TCP | 5000 | 0.0.0.0/0 |
Custom TCP | TCP | 1443 | 0.0.0.0/0 |
HTTPS | TCP | 443 | 0.0.0.0/0 |
sudo apt install mongodb
mongo admin
db.createUser({
user: '<username>',
pwd: '<password>',
roles: [ { role: "userAdminAnyDatabase", db: "admin" } ]
});
db.auth("<username>", "<password>")
use bug_db
db.createUser({
user: '<username>',
pwd: '<password>',
roles: [ { role: "readWrite", db: "bug_db" } ]
});
db.auth("<username>", "<password>")
exit
tar xvzf ./mongodata_bk.tar.gz
cd mongodata_bk
mongorestore --host 127.0.0.1 --port 27017 --username <username> --password <password> --authenticationDatabase bug_db ./
------
sudo apt install npm
Ref: https://github.com/mrvautin/adminMongo
Installation
Navigate to folder & install adminMongo: git clone https://github.com/mrvautin/adminMongo.git && cd adminMongo
Install dependencies: npm install
Start application: npm start or node app
nano app.js
change
var privateKey = fs.readFileSync( '/etc/pki/nginx/private.key' );
var certificate = fs.readFileSync( '/etc/pki/nginx/full_chain.pem' );
)
(need to update to the ssl cert generated by let's encrypt)
https://<IP>:1443/dbApp/app/login
ubuntu@ip-172-31-22-193:/home/smartscript/smartscript_web$ sudo chmod 777 -R ./
Install pip
sudo apt-get install cmake build-essential pkg-config libgoogle-perftools-dev
python3 -m pip install uwsgi torch gitpython redbaron tqdm toml sentencepiece logutils pymongo symspellpy pandas termcolor
/home/smartscript/smartscript_web pip install -r requirements pip3 install django-crontab python3 manage.py runserver 0.0.0.0:8000
ubuntu@ip-172-31-22-193:/home/smartscript/smartscript_web/smartscript$ nano settings.py
Add ip to ALLOWED_HOSTS
cd /home/smartscript/smartscript_web/py_checker python3 ./reload_daemon.py Starting httpd server on localhost:8877
pip install requirements.txt
mongodb restore the database from /home/smartscript/clcheck_db
ubuntu@ip-172-31-22-193:/home/smartscript/clcheck_db$ cd dump/
ubuntu@ip-172-31-22-193:/home/smartscript/clcheck_db/dump$ ll
total 16
drwxrwxr-x 4 ubuntu ubuntu 4096 May 6 20:04 ./
drwxrwxrwx 3 1001 1002 4096 May 6 20:04 ../
drwxrwxrwx 2 root root 4096 Aug 12 2020 clchecker/
drwxrwxrwx 2 root root 4096 Aug 12 2020 dockerfiles/
ubuntu@ip-172-31-22-193:/home/smartscript/clcheck_db/dump$
mongorestore --host 127.0.0.1 --port 27017 ./dump/
Edit /home/smartscript/clcheck/config.py
modify MONGO_URI in config.py at smartscript/clcheck/ :
MONGO_URI = os.getenv('MONGO_URI', 'mongodb:https://localhost')
Change /home/smartscript/clcheck/web/client/src/views/*.vue
Replace https://127.0.0.1:5000/clcheck/ to https://ip:5000/clcheck/
// If the server starts with SSL, then use https://smart-scripts.org:5000/clchecl/
And run 'npm install'
edit /home/smartscript/clcheck/start_web.py, app.run(debug=DEBUG, port=5000) to app.run(debug=DEBUG, host="0.0.0.0", port=5000)
Run the below command to avoid ENOSPC:
echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p
<!-- Edit "link href=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/clcheck/css/" in index.html, replace '/clcheck' with '/static/py_checker/clcheck' -->
edit /home/smartscript/clcheck/web/client$ nano vue.config.js:
publicPath: "/static/py_checker/clcheck/",
Pack up:
npm run build
copy the ./dist to /home/smartscript/smartscript_web/static/py_checker/clcheck
Backend Server side:
ubuntu@ip-172-31-22-193:/home/smartscript/clcheck$ python3 ./start_web.py
Web server: (no longer needed, just for testing, handled using ./dist folder by nginx)
ubuntu@ip-172-31-22-193:/home/smartscript/clcheck/web/client$ npm run serve
npm run serve --host=0.0.0.0
Temp key file, will be updated with "let's encrypt".
Key files:
ubuntu@ip-172-31-22-193:/home/config/cert$ ll
total 16
drwxrwxrwx 2 root root 4096 May 1 21:23 ./
drwxrwxrwx 3 root root 4096 May 1 21:28 ../
-rwxrwxrwx 1 root root 3498 May 1 21:22 fullchain.pem*
-rwxrwxrwx 1 root root 1704 May 1 21:23 privkey.pem*
ubuntu@ip-172-31-22-193:/home/config/cert$
sudo apt install nginx
sudo apt-get install libnginx-mod-http-auth-pam
mkdir /home/config and create smart-scripts_uwsgi.ini
copy the /home/pyenv_web/
for nginx, create sites-available\smart-scripts.conf under /etc/nginx
server {
listen 80;
server_name _;
charset utf-8;
client_body_timeout 300s;
uwsgi_read_timeout 3000s;
fastcgi_buffers 16 256k;
send_timeout 3m;
keepalive_timeout 3m;
fastcgi_connect_timeout 600;
fastcgi_send_timeout 600;
fastcgi_read_timeout 600;
fastcgi_buffer_size 256k;
fastcgi_busy_buffers_size 512k;
fastcgi_temp_file_write_size 512k;
error_log /var/log/nginx/error.log debug;
client_max_body_size 75m;
if ($server_port = 80 ) {
return 301 https://$host$request_uri;
}
location /media {
alias /home/smartscript/smartscript_web/media;
}
location /static {
alias /home/smartscript/smartscript_web/static;
}
location / {
uwsgi_pass unix:https:///home/config/smart-scripts.sock;
include /etc/nginx/uwsgi_params;
uwsgi_read_timeout 18000s;
proxy_read_timeout 3600;
}
}
ln -s this file to site-enable.
create /home/config/smart-scripts.sock, with chomd 777
Meanwhile, Edit /etc/nginx/nginx.conf
user www-data;
worker_processes auto;
pid /run/nginx.pid;
include /etc/nginx/modules-enabled/*.conf;
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Basic Settings
##
uwsgi_read_timeout 300s;
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
# server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
include /etc/nginx/mime.types;
default_type application/octet-stream;
##
# SSL Settings
##
# server {
# listen 443;
# server_name smart-scripts.org;
# ssl on;
# # root html;
# # index index.html index.htm;
# ssl_certificate /etc/pki/nginx/full_chain.pem;
# ssl_certificate_key /etc/pki/nginx/private.key;
# ssl_session_timeout 5m;
# ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4;
# ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
# ssl_prefer_server_ciphers on;
# location / {
#root html;
#index index.html index.htm;
# }
# }
server {
listen 443;
server_name smart-scripts.org www.smart-scripts.org;
ssl on;
charset utf-8;
client_max_body_size 75m;
ssl_certificate /etc/letsencrypt/live/smart-scripts.org-0001/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/smart-scripts.org-0001/privkey.pem; # managed by Certbot
ssl_session_timeout 5m;
ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
location /media {
alias /home/smartscript/smartscript_web/media;
}
location /static {
alias /home/smartscript/smartscript_web/static;
}
location / {
uwsgi_pass unix:https:///home/config/smart-scripts.sock;
include /etc/nginx/uwsgi_params;
}
}
server {
listen 443;
server_name db.smart-scripts.org;
ssl on;
charset utf-8;
client_max_body_size 75m;
ssl_certificate /etc/letsencrypt/live/smart-scripts.org-0001/fullchain.pem; # managed by Certbot
ssl_certificate_key /etc/letsencrypt/live/smart-scripts.org-0001/privkey.pem; # managed by Certbot
ssl_session_timeout 5m;
ssl_ciphers ECDHE-RSA-AES128-GCM-SHA256:ECDHE:ECDH:AES:HIGH:!NULL:!aNULL:!MD5:!ADH:!RC4;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
location /dbApp {
proxy_pass https://0.0.0.0:1443/dbApp;
proxy_http_version 1.1;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection 'upgrade';
proxy_set_header Host $host;
proxy_cache_bypass $http_upgrade;
}
}
ssl_protocols TLSv1 TLSv1.1 TLSv1.2; # Dropping SSLv3, ref: POODLE
ssl_prefer_server_ciphers on;
##
# Logging Settings
##
access_log /var/log/nginx/access.log;
error_log /var/log/nginx/error.log;
##
# Gzip Settings
##
gzip on;
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/javascript text/xml application/xml application/xml+rss text/javascript;
##
# Virtual Host Configs
##
include /etc/nginx/conf.d/*.conf;
include /etc/nginx/sites-enabled/*;
}
#mail {
# # See sample authentication script at:
# # https://wiki.nginx.org/ImapAuthenticateWithApachePhpScript
#
# # auth_http localhost/auth.php;
# # pop3_capabilities "TOP" "USER";
# # imap_capabilities "IMAP4rev1" "UIDPLUS";
#
# server {
# listen localhost:110;
# protocol pop3;
# proxy on;
# }
#
# server {
# listen localhost:143;
# protocol imap;
# proxy on;
# }
#}
change # managed by Certbot
to /home/config/cert
sudo apt install supervisor
under /etc/, create supervisord.conf
Think@smart-scripts:/etc$ cat /etc/supervisord.conf
; Sample supervisor config file.
;
; For more information on the config file, please see:
; https://supervisord.org/configuration.html
;
; Notes:
; - Shell expansion ("~" or "$HOME") is not supported. Environment
; variables can be expanded using this syntax: "%(ENV_HOME)s".
; - Quotes around values are not supported, except in the case of
; the environment= options as shown below.
; - Comments must have a leading space: "a=b ;comment" not "a=b;comment".
; - Command will be truncated if it looks like a config file comment, e.g.
; "command=bash -c 'foo ; bar'" will truncate to "command=bash -c 'foo ".
;
; Warning:
; Paths throughout this example file use /tmp because it is available on most
; systems. You will likely need to change these to locations more appropriate
; for your system. Some systems periodically delete older files in /tmp.
; Notably, if the socket file defined in the [unix_http_server] section below
; is deleted, supervisorctl will be unable to connect to supervisord.
[unix_http_server]
file=/tmp/supervisor.sock ; the path to the socket file
;chmod=0700 ; socket file mode (default 0700)
;chown=nobody:nogroup ; socket file uid:gid owner
;username=user ; default is no username (open server)
;password=123 ; default is no password (open server)
; Security Warning:
; The inet HTTP server is not enabled by default. The inet HTTP server is
; enabled by uncommenting the [inet_http_server] section below. The inet
; HTTP server is intended for use within a trusted environment only. It
; should only be bound to localhost or only accessible from within an
; isolated, trusted network. The inet HTTP server does not support any
; form of encryption. The inet HTTP server does not use authentication
; by default (see the username= and password= options to add authentication).
; Never expose the inet HTTP server to the public internet.
;[inet_http_server] ; inet (TCP) server disabled by default
;port=127.0.0.1:9001 ; ip_address:port specifier, *:port for all iface
;username=user ; default is no username (open server)
;password=123 ; default is no password (open server)
[supervisord]
logfile=/tmp/supervisord.log ; main log file; default $CWD/supervisord.log
logfile_maxbytes=50MB ; max main logfile bytes b4 rotation; default 50MB
logfile_backups=10 ; # of main logfile backups; 0 means none, default 10
loglevel=info ; log level; default info; others: debug,warn,trace
pidfile=/tmp/supervisord.pid ; supervisord pidfile; default supervisord.pid
nodaemon=false ; start in foreground if true; default false
minfds=1024 ; min. avail startup file descriptors; default 1024
minprocs=200 ; min. avail process descriptors;default 200
;umask=022 ; process file creation umask; default 022
;user=supervisord ; setuid to this UNIX account at startup; recommended if root
;identifier=supervisor ; supervisord identifier, default is 'supervisor'
;directory=/tmp ; default is not to cd during start
;nocleanup=true ; don't clean up tempfiles at start; default false
;childlogdir=/tmp ; 'AUTO' child log dir, default $TEMP
;environment=KEY="value" ; key value pairs to add to environment
;strip_ansi=false ; strip ansi escape codes in logs; def. false
; The rpcinterface:supervisor section must remain in the config file for
; RPC (supervisorctl/web interface) to work. Additional interfaces may be
; added by defining them in separate [rpcinterface:x] sections.
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
; The supervisorctl section configures how supervisorctl will connect to
; supervisord. configure it match the settings in either the unix_http_server
; or inet_http_server section.
[supervisorctl]
serverurl=unix:https:///tmp/supervisor.sock ; use a unix:https:// URL for a unix socket
;serverurl=https://127.0.0.1:9001 ; use an https:// url to specify an inet socket
;username=chris ; should be same as in [*_http_server] if set
;password=123 ; should be same as in [*_http_server] if set
;prompt=mysupervisor ; cmd line prompt (default "supervisor")
;history_file=~/.sc_history ; use readline history if available
; The sample program section below shows all possible program subsection values.
; Create one or more 'real' program: sections to be able to control them under
; supervisor.
;[program:theprogramname]
;command=/bin/cat ; the program (relative uses PATH, can take args)
;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
;numprocs=1 ; number of processes copies to start (def 1)
;directory=/tmp ; directory to cwd to before exec (def no cwd)
;umask=022 ; umask for process (default None)
;priority=999 ; the relative start priority (default 999)
;autostart=true ; start at supervisord start (default: true)
;startsecs=1 ; # of secs prog must stay up to be running (def. 1)
;startretries=3 ; max # of serial start failures when starting (default 3)
;autorestart=unexpected ; when to restart if exited after running (def: unexpected)
;exitcodes=0 ; 'expected' exit codes used with autorestart (default 0)
;stopsignal=QUIT ; signal used to kill process (default TERM)
;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
;stopasgroup=false ; send stop signal to the UNIX process group (default false)
;killasgroup=false ; SIGKILL the UNIX process group (def false)
;user=chrism ; setuid to this UNIX account to run the program
;redirect_stderr=true ; redirect proc stderr to stdout (default false)
;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO
;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB)
;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10)
;stdout_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
;stdout_events_enabled=false ; emit events on stdout writes (default false)
;stdout_syslog=false ; send stdout to syslog with process name (default false)
;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO
;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB)
;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10)
;stderr_capture_maxbytes=1MB ; number of bytes in 'capturemode' (default 0)
;stderr_events_enabled=false ; emit events on stderr writes (default false)
;stderr_syslog=false ; send stderr to syslog with process name (default false)
;environment=A="1",B="2" ; process environment additions (def no adds)
;serverurl=AUTO ; override serverurl computation (childutils)
; The sample eventlistener section below shows all possible eventlistener
; subsection values. Create one or more 'real' eventlistener: sections to be
; able to handle event notifications sent by supervisord.
;[eventlistener:theeventlistenername]
;command=/bin/eventlistener ; the program (relative uses PATH, can take args)
;process_name=%(program_name)s ; process_name expr (default %(program_name)s)
;numprocs=1 ; number of processes copies to start (def 1)
;events=EVENT ; event notif. types to subscribe to (req'd)
;buffer_size=10 ; event buffer queue size (default 10)
;directory=/tmp ; directory to cwd to before exec (def no cwd)
;umask=022 ; umask for process (default None)
;priority=-1 ; the relative start priority (default -1)
;autostart=true ; start at supervisord start (default: true)
;startsecs=1 ; # of secs prog must stay up to be running (def. 1)
;startretries=3 ; max # of serial start failures when starting (default 3)
;autorestart=unexpected ; autorestart if exited after running (def: unexpected)
;exitcodes=0 ; 'expected' exit codes used with autorestart (default 0)
;stopsignal=QUIT ; signal used to kill process (default TERM)
;stopwaitsecs=10 ; max num secs to wait b4 SIGKILL (default 10)
;stopasgroup=false ; send stop signal to the UNIX process group (default false)
;killasgroup=false ; SIGKILL the UNIX process group (def false)
;user=chrism ; setuid to this UNIX account to run the program
;redirect_stderr=false ; redirect_stderr=true is not allowed for eventlisteners
;stdout_logfile=/a/path ; stdout log path, NONE for none; default AUTO
;stdout_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB)
;stdout_logfile_backups=10 ; # of stdout logfile backups (0 means none, default 10)
;stdout_events_enabled=false ; emit events on stdout writes (default false)
;stdout_syslog=false ; send stdout to syslog with process name (default false)
;stderr_logfile=/a/path ; stderr log path, NONE for none; default AUTO
;stderr_logfile_maxbytes=1MB ; max # logfile bytes b4 rotation (default 50MB)
;stderr_logfile_backups=10 ; # of stderr logfile backups (0 means none, default 10)
;stderr_events_enabled=false ; emit events on stderr writes (default false)
;stderr_syslog=false ; send stderr to syslog with process name (default false)
;environment=A="1",B="2" ; process environment additions
;serverurl=AUTO ; override serverurl computation (childutils)
; The sample group section below shows all possible group values. Create one
; or more 'real' group: sections to create "heterogeneous" process groups.
;[group:thegroupname]
;programs=progname1,progname2 ; each refers to 'x' in [program:x] definitions
;priority=999 ; the relative start priority (default 999)
; The [include] section can just contain the "files" setting. This
; setting can list multiple files (separated by whitespace or
; newlines). It can also contain wildcards. The filenames are
; interpreted as relative to this file. Included files *cannot*
; include files themselves.
;[include]
;files = relative/directory/*.ini
[program:smart-scripts]
command=/usr/local/bin/uwsgi --ini /home/config/smart-scripts_uwsgi.ini
directory=/home/smartscript/smartscript_web
startsecs=0
stopwaitsecs=0
autostart=true
autorestart=true
stdout_logfile=/home/config/supervisor.out.log
stderr_logfile=/home/config/supervisor.err.log
stopsignal=QUIT
killasgroup=true
Think@smart-scripts:/etc$
sudo npm install forever -g
/etc/rc.local:
Old file:
#!/bin/bash
/usr/local/bin/supervisord -c /etc/supervisord.conf
# sudo docker start smartscript_db
sudo touch /home/doing
sudo chmod 777 /home/config/smart-scripts.sock
sudo forever start /home/GitRepo/adminMongo/app.js
sudo service nginx restart
exit 0
https://manual.uberspace.de/daemons-supervisord/
Update location of uwsgi: Modify /etc/supervisord.conf
and execute:
$ supervisorctl reread
And then start your daemon:
$ supervisorctl update
supervisorctl stop all
smart-scripts: stopped
ubuntu@ip-172-31-22-193:/home/smartscript/adminMongo$ cat /home/config/smart-scripts_uwsgi.ini
[uwsgi]
chdir = /home/smartscript/smartscript_web
module = smartscript.wsgi
home = /home/pyenv_web/
logto = /home/config/uwsgi.log
# master = true
processes = 4
socket = /home/config/smart-scripts.sock
# shared-socket = 0.0.0.0:443
chmod-socket = 664
vacuum = true
# https = =0,/home/config/cert/full_chain.pem,/home/config/cert/private.key,HIGH
# http-to = /home/config/smart-scripts.sock
ubuntu@ip-172-31-22-193:/home/smartscript/adminMongo$ /usr/local/bin/uwsgi
-bash: /usr/local/bin/uwsgi: No such file or directory
ubuntu@ip-172-31-22-193:/home/smartscript/adminMongo$ which uwsgi
/home/ubuntu/.local/bin/uwsgi
New file /etc/rc.local:
#!/bin/bash
/usr/bin/supervisord -c /etc/supervisord.conf
sudo touch /home/doing
sudo chmod 777 /home/config/smart-scripts.sock
sudo forever start /home/smartscript/adminMongo/app.js
sudo service nginx restart
exit 0
restart nginx
error log for debugging: cat /var/log/nginx/error.log
- We need to change the domain of this website to our IP address for achieving https features.
Follow the instruction on https://certbot.eff.org/lets-encrypt/ubuntubionic-nginx
sudo certbot --nginx
Don't forget to update the app.js in adminMongo
sudo swapon --show
sudo fallocate -l 20G /swapfile
sudo chmod 600 /swapfile
sudo mkswap /swapfile
sudo swapon /swapfile
sudo free -h
sudo bash /etc/rc.local
screen: ubuntu@ip-172-31-22-193:/home/smartscript/clcheck$ python3 ./start_web.py
ubuntu@ip-172-31-22-193:/home/smartscript/smartscript_web/py_checker$ python3 ./reload_daemon.py