Unverified Commit 939d9aba authored by Mark Beamer Jr's avatar Mark Beamer Jr
Browse files

Add CleanOnStartup for local testing and e2e testing

Update methods to be refactored into util package to be used in many locations.
Add end to end test of ytsync for CI
parent 131baa36
os: linux
dist: xenial
language: go
go_import_path: github.com/lbryio/ytsync
go:
- 1.11.5
- master
- 1.12.1
- master
services:
- docker
env:
global:
- GO111MODULE=on
#GITHUB_TOKEN
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
- secure: Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI=
#SLACK_TOKEN
- secure: iCAZGBjwyNMlh2U3guni+TZ0LGtcfvXDxit6/W4ZytO1Px+RGwPAn6hQ49LQAKPhj8LBfYYHZzjRWtuTIqSoxeP7WumP/PknlNzgxjxCQva489rQsHzP+xZztm/dQNdAJAQi/xkRDz2Yd/TH8EFasKGnwwyQrutt6s8efwb/y4OIMDFJEyH2lZwvveVz3xdOlkRyYdJuUFr98OWqPLaUKpd8eKrnKoSHemJ2zMw3a3ozOnXcMTUkN57jclcy+JQTU4qHuebVYAM3gkxjVOoSLLw+si3xhuNsUGpH5VNT4P926nx5N3u1P549yHm6XSPnrVdQb3F0M1oQXOo92R87ASAx2QQ/6riIzcqqahE7Ljz3wEtTjezYfoSCRIGoZG8WbVSb1psmen546npwfNGEZZxLNTFHnQIMGoU2ePE1rui+OcdLYe06ibRaKS0uTMuPiYQmzYegFYHws0DcRLrbCJklOt4SRzQnW+25dAW8scXV9yOuSqMP/OuWNu6rFrcsOgSvQ4yp50c5SDa3HCalsCTw3UTffDG/BDf10PsG6BP/nRa9RT3mNw4NIDcKcrl3ZfD4ZMtTuFLeNd9rQXVRIvGFiCV0zawSx5sNNHvuovHmxuKXIljnmOzZG7XhShxmbIh5oB3uLCvCNNWQZtHW2DWxeodC2INENs479SOjyqg=
#YOUTUBE_API_KEY
- secure: MWM08SbjWSP74bWuP8uuqTYDF0cHKCruLiXXpeTHTpDsoXWqHwUIy7Mm5urDk/UPcd/2x7dqUWjPTyRhx61zgQsgUgPU9iWaB9hQTE9wfMp5vT8dwwsHjexO3fw1R+gyTcJBcsWG9rOSBY45+zFYQtzI7j/XyNAg/ifCQFwkC89PjGZfWggUXtuSg6aPbBzyFRqH7xbePoMwufU9Ktczb8SNMzeewatiohe5gMRW2LXoUXmJs/uGQUklIzf1hxTItRdvblOeE9VDBFXd5Q0Q1zb626sEXKvIkEQAhWHd7Q1YqfXu4CNt2vUt+h8FOC+2SGMaBRC6DaqJh7fyf6ZaxgsO8Uyv/dBgUxwOdLMvyDF6zEnQkOLYbJcaqcou4OF77L5i/jvUGfnUw/MUQnuN8Xt/lzXHYbWExrmsvFDCN+8zaLxgqtgD8CcTNwaf9bLSJGF+WqYaXLw+wUXXH6COWUceDNf5hqh1W0TfOewA0WeB0DcD+q7PowmYRwXdkoGNhV5elAJ1/4s8LmoeMuo6PHBsE433p0TnaomQVshUTfrFnvYGmKxIfGNe8Qz4Rgqka0TFg06zMLDuhHT3Eev53bO5Dr2dySM7yf9MKCTwqj18LlFvelq0fUK1j7GySj8Gi3eulBpmHexUuj6UIqgeBKMADlQCxTnFmZ8RNVRT1pY=
#AWS_S3_ID
- secure: 97fr5cl6/pnCGUsnB3/nSgGEW3tn0MJWnosNO4k9yhnQOSxBJ1MZzLPVa79WD26ekzyfeCgEge3/S4Q1PyOAHvMxSIklrAxReOMSVfjRE8zzySbeiqsQ5MUDholeQ4Rrx0p2aXNbr9W5hO5iIE+oOvgUS4Rct4uZCeaxOq8YIPIpExKLtCZpapHae/cOoGM7ThRIFXXiQRvB2zIcoGcUF9y/6viW0HHvtJbe0NKkmbUDnsQb6K3jMBjZJ/Gmg52Ix+/qe2iD0/Qr8/FQcciqddG8IOA/UcQq4hN817ofTKQ6XID8qAfZAWjvSp7twk4ym44HZ/ORdRHMTVAbVzV67bL1QtOtXEg7W2nLnmydxGpBOObs1nZbfIpKz48BHH9PsLDi3dbs2X/98jOVUDQL1qqin72tpSv87tp43umfcjRtN0RV6K4SKMG3bRC7JCpjEImo4XkGxlTGZQkusXktTbx/H3UmssUFMlu8SQWbcoRNgb3mLRmxx2DWyJtWEBNJgo3aXxJCeEhvWP6odY+RSeI2axS3tX//aCdvuuA1gV5zV0tD4Q7jQRDF8ZVfPkn1f/rvwZs+8L/etJnCBXvEmtuysJDfeVXWWxcExNfo9ckUNJlk8yu8immi27ZkdQMrUUjk4GeEKlRKNjyWxHVm9sKslmYaeg2GuY6T5lex9xc=
#AWS_S3_SECRET
- secure: vRfVtuV1iyXwwMxm/PfyOW8WOk1JFDWWQuBkAHqCfgWxyl8+WOHQKTpDyn5HtnVkk/JDie3fBqno3x031pG/fJgnByeptXYE+++IvShMtsayK+7bTNVGp3sZCmomO+Yq4qjV/y+DFDgvvxqSW0JFEfOO2lBCOZM5a0TSRbaXgPg28wTPMXZIhg0qZtneXskMg7afH6it5TuOM/Hu1GIhbgOKzQEV2KOqC/GoyG70lw2bZGnzL/TJuvWuODQCobpwVj+PTWG0+LXKDXxA/27mbmdXQYdC/tSK/xvL5vH8vjqSe7nVYA0eCpRmAhLYF5GZR5IlewCrIvjHsKOquaDBKIKOlnV9ZAzAQYeg5vTlWPNReKeLs7UM4QOhCpyrSt22yvHHWCN8QN2kzey36zJ/CpxweqAKOKZbSAJgUZwnjx/LEGv9BV6RPaNgCWD9SVg2nJ8Ab17ZGnMUT0QL6Gjl5Y4vJsnOVi6IptoAJmUTAndK+HMxoTduEoUdFv1OtRU3pxTZ178upowwbwwkGclngu6uwb/+jRgSB3qyC8Yjmi3SI1jARCag2Ah+Tg+Hdu7ucGCWdI+P2UCSHxSItiRcTIUTfoagdBoCU4CJUGTX+Pd58b3b+63tmDGrCW5SeIGqe64tutq6x6aGZ2zh2La8SlIYBdJwocYe+8CvjbD61lk=
#AWS_S3_REGION
- secure: HvWilVKzfbMuonKUBY3my7JTYxcPXSPVdeyBfrm4J42QPWFqKAk+i3yjHJWs/X7LJix6tOclYnznXBUBLbsrOGOafLoyoOtGJkkMF2th/oWti7rgsJJ0F5BMB3+v5UaD/xEBSlBRC+q6mqo+BF0VDCHybfkjQ6zR+IzInjNaVZPVDMZ/3fOIUoQ2YOZcNr6waHtzxP3UFRwifjJDfa5iNbo77BtuJ1DH6bZVExpVO18nmxhUUBDmmwAn/oJ7UL0xQhRuop0lD2yih7Xxfxby7Q9qGHrfBPaKeSlwwQ0i0mqixrrfN7Ru/aU7WJ0aPZyi1UbYraGtW4/uUUbacDqZCza7fXih3LI6J0//tPT0dg/PI8PFZuzFRQF4x5Tatbv4ePNrio9T0xKM8nDiCjOG6bVm/JwLLaLp2uvbPrWkiKbE2Nyo/rScD+tU+0xEnDU98DP8lTK60vDVviyLalYRkoP3v89LUJ9S8OocdbRm3erhvdHXqAcAOSsMwTVDvsZV9cb2Zy/4/29m+Y8PeRcSrcCuw71suAoBmjJXsQovwhSyezmDyqvDCgaqoJc9VyUNHrQKWQCJFVC+4jAlRVB2A5YcrU1vJwARkdHGQP2fcfu58U+ErXuODKlxub4ZijI8cnb9+SPbYfn38Mv8bWsjh2QkDCpVky/nUPV2NflxRjc=
#AWS_S3_BUCKET
- secure: xjnqd7FocCsBUkPKdHAwCUqLI7OQpIRn4fmJgu84JvRYdky5RqS+ZaTSv4YMqqTG3Mad0E/brEKMx4K3swErPtCzLal76Zxb9oOnkvUB5MqQ3FffftHAG26x5D0Ja7lI5QqwGrfg7lfBDcJXlIPTP86ZuRJ6uwn42vjFBOQrAESSoaHI9hHMwM6aGvQGtUPyyAPTqlu7ATgBWoT9/+MVxiuiEuoNDE40XehL2Cmu9eP5mL/A0ucG/YaUhIEzvPUYh8oF9JGariSG5ilj8DZiveN+j5lyFKMSy0S6qyjuZVSWPM7rr7a3oxrrMKbVa/nQkSa9/DnHoVa/XOZDqXhR62CE+aF0Ay0VtrGKT3NxsRpCoX/QvvWVCPKrTx6Ps4ZxTlOUTsG52OpuVdIf07rfB2tR3R7BOSs0xBl3d/FW1ZitNVe1tQu75tbwg2XF5oVVXGvyKIJaLUgP5wYgm2iGaMDYPtp7SLUJmEgvw2Ppr/VxcfI2OSyE7PhoXYzguvUKEBf4FsfivJEfIphLq5dMu6hYxHgZ3huc+duh6L+uPEetpZI9VQKaj9OvPitK9mgsGshr4twE/yl8IL998EjkxuDZmqYvv4kwj09f1TJMpQeo8tP2PaEY07Lcd+sNSE7QuUVPBbM4KFkiZXz9w1Nsl3RBVIPZbAEzmIR15C9o5SU=
#GOOGLE_ID
- secure: sx5QXSTOcklS8F0q3PvFQsyME/0S/tORYCv1SNKAPlbzVCkPuYIjM5/gUoaYyFoo+dguEO5OsaeCyIf1sAbg8xQlonuPalIxJT+hU03yTtZNPaTGgXMbbQ2R8lYzrOaKaHxPp36a5ihv96BxCW4ssBjOcPoCz/Ae2v9XPFeZDJrhMfwwoo8rDrLDj90uBpsSIK22rcBzXRpi1t36xUjE3ri2cMddQnXIrX/p8aiDcYMi+a4EqwRdtC7/VAB3a+V74uqC7vpAuhzA0YEe4U1K0V86Z0/CAYx0TD9KWt2EuImgcfgFo/8272wm+nRzK+hP44sjjal+3Y1KkvzJyBK/yszYL2niGbm4HsPHIaDBBmKODR5i7hn0HnjJ/6GHFe6X2oSHSvJVe0K7pFN0wcYIzLBdPh9jv7yX5/3oih8RuqqUKQAynmgROuiAQ8VuYu9nUjsE/Zi0B7GX5nPJ4zWtCGlblAPa+CQTYETE1g/TB7xut7GVsJJj2J8izDevypEx8l3oJsTEBxMpaKZJvoOBUnjJJOeD4gcmtiljhjQw0D+TfyRkU/JJHLJJ02YTyJfxAJ8eOCeE1wSx0lg+btcxSav/0Q4uK+L8ifF6LzFnDBl6q8OCJHe0BEppxWNFr91I/ChC/sCyR7gWmhbEj1AiENSVLZv0jl+hRPp5qKgBwNE=
#GOOGLE_TOKEN
- secure: 1yRf7EebS0XSy59q5vINFgqxEwdZKNK3/cdbWAMZ7DvZsvYkeoOrgHAzxUi+BkuLT0cnObpceQKGko47dD7If0IsKTO/N/91KVOD5vCNz/bLM6MSjf46Ien/S8MXfLfWux566IAVocdoVZghAjhwj80uSvn+Lp8LMBhnFrwf7x+f2AwLsMBhybwI+5h2WXE654zNAgnHIzud3AAURN0IFsRkXNLEd6ZH4w6Fg2SCHlCcaxd/BYQ4djUgIvRXAgCL7xSKsCnNxg/AqkkWKPQ5gu02FZLXXW0M1j+018PLJ7puRp4XQgnDK/xbD5y7mv/AGC1w2oThcZeflmY+5txd/4c4ps9B3eJdGXMLPIIg+v2XIqSjYhMUjg9y0TLQsQsih3fBrHOymZwcl06+e8eWfVQ6OkW2/7SGXdCJXwTrm8Mrx1uhmWrg/syd1sIfu5xHqxIZk51QUeXNyN0k1B42BlAznEWahif1iS+vdlD8EiKqaIRnMe1U9jHkcsXFebtirdF2i/FCSgnjXGFo1H0MXq866FDkeP+bw+5YSG6CsWEu4ovq37V8jzLBu9vyodOW40Wws03Xc7DtxjNHlYrhqOfA+x+jmokxcpKtaFHk8pBxlPxVZArTmEO3je8QgdHeM505tBfbBtprgTfZHMzJmaVqhVq65Bl3foG987eFVpw=
script:
- ./e2e/e2e.sh
deploy:
provider: script
skip_cleanup: true
......
......@@ -13,7 +13,6 @@ import (
"github.com/lbryio/reflector.go/reflector"
"github.com/lbryio/reflector.go/store"
"github.com/lbryio/ytsync/util"
"github.com/mitchellh/go-ps"
log "github.com/sirupsen/logrus"
)
......@@ -23,7 +22,7 @@ func ReflectAndClean() error {
if err != nil {
return err
}
return cleanupLbrynet()
return util.CleanupLbrynet()
}
func loadConfig(path string) (cmd.Config, error) {
......@@ -46,7 +45,7 @@ func reflectBlobs() error {
return nil
}
//make sure lbrynet is off
running, err := isLbrynetRunning()
running, err := util.IsLbrynetRunning()
if err != nil {
return err
}
......@@ -95,60 +94,3 @@ func reflectBlobs() error {
}
return nil
}
func cleanupLbrynet() error {
//make sure lbrynet is off
running, err := isLbrynetRunning()
if err != nil {
return err
}
if running {
return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
}
lbrynetDir := util.GetLBRYNetDir()
files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
if err != nil {
return errors.Err(err)
}
for _, f := range files {
err = os.Remove(f)
if err != nil {
return errors.Err(err)
}
}
blobsDir := util.GetBlobsDir()
err = os.RemoveAll(blobsDir)
if err != nil {
return errors.Err(err)
}
err = os.Mkdir(blobsDir, 0755)
if err != nil {
return errors.Err(err)
}
return nil
}
func isLbrynetRunning() (bool, error) {
if util.IsUsingDocker() {
container, err := util.GetLBRYNetContainer(util.ONLINE)
if err != nil {
return false, err
}
return container != nil, nil
}
processes, err := ps.Processes()
if err != nil {
return true, errors.Err(err)
}
var daemonProcessId = -1
for _, p := range processes {
if p.Executable() == "lbrynet" {
daemonProcessId = p.Pid()
break
}
}
running := daemonProcessId != -1
return running, nil
}
......@@ -33,6 +33,8 @@ services:
- TCP_PORT=50001
ports:
- "15300:50001"
depends_on:
- lbrycrd
ulimits:
nofile: 90000
# command: lbry.wallet.server.coin.LBC
......@@ -46,10 +48,45 @@ services:
ports:
- "15100:5279"
- "15101:5280"
depends_on:
- walletserver
environment:
- LBRY_STREAMING_SERVER=0.0.0.0:5280
volumes:
- "./persist/.lbrynet:/home/lbrynet"
- ".:/etc/lbry" #Put your daemon_settings.yml here
# /private/var/tmp for OSX and /var/tmp for Linux
- "/private/var/tmp:/var/tmp"
- "${LOCAL_TMP_DIR}:/var/tmp"
#entrypoint: wait-for-it walletserver:50001 -- start
###########
## MySQL ##
###########
mysql:
image: mysql/mysql-server:5.7.27
restart: "no"
ports:
- "15500:3306"
environment:
- MYSQL_ALLOW_EMPTY_PASSWORD=true
- MYSQL_DATABASE=lbry
- MYSQL_USER=lbry
- MYSQL_PASSWORD=lbry
- MYSQL_LOG_CONSOLE=true
###################
## Internal APIs ##
###################
internalapis:
image: lbry/internal-apis:latest
restart: "no"
ports:
- "15400:8080"
depends_on:
- mysql
- lbrycrd
environment:
- MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
- LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
- MYSQL_USER=lbry
- MYSQL_PASS=lbry
- MYSQL_DATABASE=lbry
entrypoint: wait-for-it mysql:3306 -- ./latest serve
#!/usr/bin/env bash
set -e
#Always compile ytsync
make -C ./..
#OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details
export LOCAL_TMP_DIR="/var/tmp"
#Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET
touch -a .env && set -o allexport; source ./../.env; set +o allexport
echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR"
# Compose settings - docker only
export SLACK_CHANNEL="ytsync-travis"
export LBRY_API_TOKEN="ytsyntoken"
export LBRY_WEB_API="http://localhost:15400"
export LBRYNET_ADDRESS="http://localhost:15100"
export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200"
export LBRYNET_USE_DOCKER=true
export REFLECT_BLOBS=false
export CLEAN_ON_STARTUP=true
export REGTEST=true
# Local settings
export BLOBS_DIRECTORY="./blobsfiles"
export LBRYNET_DIR="`pwd`/persist/.lbrynet/.local/share/lbry/lbrynet/"
export LBRYNET_WALLETS_DIR="`pwd`/persist/.lbrynet/.local/share/lbry/lbryum"
export TMP_DIR="/var/tmp"
docker-compose down
rm -dr ./persist
docker-compose up -d
sleep 15
#Data Setup for test
#Add a ytsync user
ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
#Insert an auth token for the user to be used by ytsync
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
#Give priveledges to ytsync user
ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
#Add youtuber to sync
ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
#Add their youtube channel to be synced
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,google_id,google_token,created_at,source,total_videos,total_subscribers)
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued',$GOOGLE_ID,'$GOOGLE_TOKEN','2019-08-01 00:00:00','sync',1,0)"
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
# Execute the test!
./../bin/ytsync --channelID="UCCyr5j8akeu9j4Q7urV0Lqw" #Force channel intended...just in case. This channel lines up with the api container
\ No newline at end of file
#!/usr/bin/env bash
while true; do
lbrycrd-cli -conf=/data/.lbrycrd/lbrycrd.conf generate 1 >> /tmp/output.log
lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 100 >> /tmp/output.log
sleep 2
done
\ No newline at end of file
#!/bin/bash
if [ $# -eq 0 ]
then
echo "No docker tag argument supplied. Use './build.sh <tag>'"
exit 1
fi
docker build --tag lbry/lbrycrd:$1 .
docker push lbry/lbrycrd:$1
\ No newline at end of file
## This base image is for running the latest lbrynet-daemon release.
FROM ubuntu:18.04 as prep
LABEL MAINTAINER="leopere [at] nixc [dot] us"
RUN apt-get update && apt-get -y install unzip curl telnet
RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it
## Add lbrynet
ARG VERSION="v0.38.5"
......
#!/bin/bash
if [ $# -eq 0 ]
then
echo "No docker tag argument supplied. Use './build.sh <tag>'"
exit 1
fi
docker build --tag lbry/lbrynet:$1 .
docker push lbry/lbrynet:$1
\ No newline at end of file
......@@ -101,6 +101,10 @@ const (
func (s *SyncManager) Start() error {
if logUtils.IsCleanOnStartup() {
logUtils.CleanForStartup()
}
syncCount := 0
for {
err := s.checkUsedSpace()
......
package manager
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"os"
"os/exec"
"os/signal"
"runtime/debug"
"sort"
......@@ -32,8 +30,6 @@ import (
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
"github.com/mitchellh/go-ps"
log "github.com/sirupsen/logrus"
"google.golang.org/api/googleapi/transport"
......@@ -187,18 +183,12 @@ func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string,
}
func (s *Sync) uploadWallet() error {
defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
defaultWalletDir := logUtils.GetDefaultWalletPath()
key := aws.String("/wallets/" + s.YoutubeChannelID)
if os.Getenv("REGTEST") == "true" {
defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
key = aws.String("/regtest/" + s.YoutubeChannelID)
}
walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
if walletPath != "" {
defaultWalletDir = walletPath + "/wallets/default_wallet"
}
if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
return errors.Err("default_wallet does not exist")
}
......@@ -295,7 +285,7 @@ func (s *Sync) FullCycle() (e error) {
defer deleteSyncFolder(s.videoDirectory)
log.Printf("Starting daemon")
err = startDaemon()
err = logUtils.StartDaemon()
if err != nil {
return err
}
......@@ -363,7 +353,7 @@ func (s *Sync) waitForDaemonStart() error {
func (s *Sync) stopAndUploadWallet(e *error) {
log.Printf("Stopping daemon")
shutdownErr := stopDaemon()
shutdownErr := logUtils.StopDaemon()
if shutdownErr != nil {
logShutdownError(shutdownErr)
} else {
......@@ -946,74 +936,6 @@ func (s *Sync) processVideo(v video) (err error) {
return nil
}
func startDaemon() error {
if logUtils.IsUsingDocker() {
return startDaemonViaDocker()
}
return startDaemonViaSystemd()
}
func stopDaemon() error {
if logUtils.IsUsingDocker() {
return stopDaemonViaDocker()
}
return stopDaemonViaSystemd()
}
func startDaemonViaDocker() error {
container, err := logUtils.GetLBRYNetContainer(true)
if err != nil {
return err
}
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
if err != nil {
return errors.Err(err)
}
return nil
}
func stopDaemonViaDocker() error {
container, err := logUtils.GetLBRYNetContainer(false)
if err != nil {
return err
}
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
err = cli.ContainerStop(context.Background(), container.ID, nil)
if err != nil {
return errors.Err(err)
}
return nil
}
func startDaemonViaSystemd() error {
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
if err != nil {
return errors.Err(err)
}
return nil
}
func stopDaemonViaSystemd() error {
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
if err != nil {
return errors.Err(err)
}
return nil
}
// waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
func waitForDaemonProcess(timeout time.Duration) error {
processes, err := ps.Processes()
......
......@@ -4,7 +4,9 @@ import (
"context"
"fmt"
"os"
"os/exec"
"os/user"
"path/filepath"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/lbry.go/lbrycrd"
......@@ -12,6 +14,7 @@ import (
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/mitchellh/go-ps"
log "github.com/sirupsen/logrus"
)
......@@ -102,3 +105,187 @@ func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) {
return lbrycrdd, nil
}
func IsCleanOnStartup() bool {
return os.Getenv("CLEAN_ON_STARTUP") == "true"
}
func IsLbrynetRunning() (bool, error) {
if IsUsingDocker() {
container, err := GetLBRYNetContainer(ONLINE)
if err != nil {
return false, err
}
return container != nil, nil
}
processes, err := ps.Processes()
if err != nil {
return true, errors.Err(err)
}
var daemonProcessId = -1
for _, p := range processes {
if p.Executable() == "lbrynet" {
daemonProcessId = p.Pid()
break
}
}
running := daemonProcessId != -1
return running, nil
}
func CleanForStartup() error {
if !IsRegTest() {
return errors.Err("never cleanup wallet outside of regtest and with caution. this should only be done in local testing and requires regtest to be on")
}
running, err := IsLbrynetRunning()
if err != nil {
return err
}
if running {
err := StopDaemon()
if err != nil {
return err
}
}
err = CleanupLbrynet()
if err != nil {
return err
}
lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
if err != nil {
return errors.Prefix("error getting lbrycrd client: ", err)
}
height, err := lbrycrd.GetBlockCount()
if err != nil {
return errors.Err(err)
}
if height < 110 {
//Start reg test will some credits
txs, err := lbrycrd.Generate(uint32(110) - uint32(height))
if err != nil {
return errors.Err(err)
}
log.Debugf("REGTEST: Generated %d transactions to get some LBC!", len(txs))
}
defaultWalletDir := GetDefaultWalletPath()
return os.Remove(defaultWalletDir)
}
func CleanupLbrynet() error {
//make sure lbrynet is off
running, err := IsLbrynetRunning()
if err != nil {
return err
}
if running {
return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
}
lbrynetDir := GetLBRYNetDir()
files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
if err != nil {
return errors.Err(err)
}
for _, f := range files {
err = os.Remove(f)
if err != nil {
return errors.Err(err)
}
}
blobsDir := GetBlobsDir()
err = os.RemoveAll(blobsDir)
if err != nil {
return errors.Err(err)
}
err = os.Mkdir(blobsDir, 0755)
if err != nil {
return errors.Err(err)
}
return nil
}
func StartDaemon() error {
if IsUsingDocker() {
return startDaemonViaDocker()
}
return startDaemonViaSystemd()
}
func StopDaemon() error {
if IsUsingDocker() {
return stopDaemonViaDocker()
}
return stopDaemonViaSystemd()
}
func startDaemonViaDocker() error {
container, err := GetLBRYNetContainer(true)
if err != nil {
return err
}
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
if err != nil {
return errors.Err(err)
}
return nil
}
func stopDaemonViaDocker() error {
container, err := GetLBRYNetContainer(ONLINE)
if err != nil {
return err
}
cli, err := client.NewEnvClient()
if err != nil {
panic(err)
}
err = cli.ContainerStop(context.Background(), container.ID, nil)
if err != nil {
return errors.Err(err)
}
return nil
}
func startDaemonViaSystemd() error {
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
if err != nil {
return errors.Err(err)
}
return nil
}
func stopDaemonViaSystemd() error {
err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
if err != nil {
return errors.Err(err)
}
return nil
}
func GetDefaultWalletPath() string {
defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
if os.Getenv("REGTEST") == "true" {
defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
}
walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
if walletPath != "" {
defaultWalletDir = walletPath + "/wallets/default_wallet"
}
return defaultWalletDir
}