Add CleanOnStartup for local testing and e2e testing

Update methods to be refactored into util package to be used in many locations.
Add end to end test of ytsync for CI

.travis.yml

 os: linux
 dist: xenial
 language: go
-go_import_path: github.com/lbryio/ytsync
 go:
-- 1.11.5
-- master
+  - 1.12.1
+  - master
+services:
+  - docker
 env:
   global:
-    - GO111MODULE=on
     #GITHUB_TOKEN
-    - secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
+    - secure: Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI=
+    #SLACK_TOKEN
+    - secure: iCAZGBjwyNMlh2U3guni+TZ0LGtcfvXDxit6/W4ZytO1Px+RGwPAn6hQ49LQAKPhj8LBfYYHZzjRWtuTIqSoxeP7WumP/PknlNzgxjxCQva489rQsHzP+xZztm/dQNdAJAQi/xkRDz2Yd/TH8EFasKGnwwyQrutt6s8efwb/y4OIMDFJEyH2lZwvveVz3xdOlkRyYdJuUFr98OWqPLaUKpd8eKrnKoSHemJ2zMw3a3ozOnXcMTUkN57jclcy+JQTU4qHuebVYAM3gkxjVOoSLLw+si3xhuNsUGpH5VNT4P926nx5N3u1P549yHm6XSPnrVdQb3F0M1oQXOo92R87ASAx2QQ/6riIzcqqahE7Ljz3wEtTjezYfoSCRIGoZG8WbVSb1psmen546npwfNGEZZxLNTFHnQIMGoU2ePE1rui+OcdLYe06ibRaKS0uTMuPiYQmzYegFYHws0DcRLrbCJklOt4SRzQnW+25dAW8scXV9yOuSqMP/OuWNu6rFrcsOgSvQ4yp50c5SDa3HCalsCTw3UTffDG/BDf10PsG6BP/nRa9RT3mNw4NIDcKcrl3ZfD4ZMtTuFLeNd9rQXVRIvGFiCV0zawSx5sNNHvuovHmxuKXIljnmOzZG7XhShxmbIh5oB3uLCvCNNWQZtHW2DWxeodC2INENs479SOjyqg=
+    #YOUTUBE_API_KEY
+    - secure: MWM08SbjWSP74bWuP8uuqTYDF0cHKCruLiXXpeTHTpDsoXWqHwUIy7Mm5urDk/UPcd/2x7dqUWjPTyRhx61zgQsgUgPU9iWaB9hQTE9wfMp5vT8dwwsHjexO3fw1R+gyTcJBcsWG9rOSBY45+zFYQtzI7j/XyNAg/ifCQFwkC89PjGZfWggUXtuSg6aPbBzyFRqH7xbePoMwufU9Ktczb8SNMzeewatiohe5gMRW2LXoUXmJs/uGQUklIzf1hxTItRdvblOeE9VDBFXd5Q0Q1zb626sEXKvIkEQAhWHd7Q1YqfXu4CNt2vUt+h8FOC+2SGMaBRC6DaqJh7fyf6ZaxgsO8Uyv/dBgUxwOdLMvyDF6zEnQkOLYbJcaqcou4OF77L5i/jvUGfnUw/MUQnuN8Xt/lzXHYbWExrmsvFDCN+8zaLxgqtgD8CcTNwaf9bLSJGF+WqYaXLw+wUXXH6COWUceDNf5hqh1W0TfOewA0WeB0DcD+q7PowmYRwXdkoGNhV5elAJ1/4s8LmoeMuo6PHBsE433p0TnaomQVshUTfrFnvYGmKxIfGNe8Qz4Rgqka0TFg06zMLDuhHT3Eev53bO5Dr2dySM7yf9MKCTwqj18LlFvelq0fUK1j7GySj8Gi3eulBpmHexUuj6UIqgeBKMADlQCxTnFmZ8RNVRT1pY=
+    #AWS_S3_ID
+    - secure: 97fr5cl6/pnCGUsnB3/nSgGEW3tn0MJWnosNO4k9yhnQOSxBJ1MZzLPVa79WD26ekzyfeCgEge3/S4Q1PyOAHvMxSIklrAxReOMSVfjRE8zzySbeiqsQ5MUDholeQ4Rrx0p2aXNbr9W5hO5iIE+oOvgUS4Rct4uZCeaxOq8YIPIpExKLtCZpapHae/cOoGM7ThRIFXXiQRvB2zIcoGcUF9y/6viW0HHvtJbe0NKkmbUDnsQb6K3jMBjZJ/Gmg52Ix+/qe2iD0/Qr8/FQcciqddG8IOA/UcQq4hN817ofTKQ6XID8qAfZAWjvSp7twk4ym44HZ/ORdRHMTVAbVzV67bL1QtOtXEg7W2nLnmydxGpBOObs1nZbfIpKz48BHH9PsLDi3dbs2X/98jOVUDQL1qqin72tpSv87tp43umfcjRtN0RV6K4SKMG3bRC7JCpjEImo4XkGxlTGZQkusXktTbx/H3UmssUFMlu8SQWbcoRNgb3mLRmxx2DWyJtWEBNJgo3aXxJCeEhvWP6odY+RSeI2axS3tX//aCdvuuA1gV5zV0tD4Q7jQRDF8ZVfPkn1f/rvwZs+8L/etJnCBXvEmtuysJDfeVXWWxcExNfo9ckUNJlk8yu8immi27ZkdQMrUUjk4GeEKlRKNjyWxHVm9sKslmYaeg2GuY6T5lex9xc=
+    #AWS_S3_SECRET
+    - secure: vRfVtuV1iyXwwMxm/PfyOW8WOk1JFDWWQuBkAHqCfgWxyl8+WOHQKTpDyn5HtnVkk/JDie3fBqno3x031pG/fJgnByeptXYE+++IvShMtsayK+7bTNVGp3sZCmomO+Yq4qjV/y+DFDgvvxqSW0JFEfOO2lBCOZM5a0TSRbaXgPg28wTPMXZIhg0qZtneXskMg7afH6it5TuOM/Hu1GIhbgOKzQEV2KOqC/GoyG70lw2bZGnzL/TJuvWuODQCobpwVj+PTWG0+LXKDXxA/27mbmdXQYdC/tSK/xvL5vH8vjqSe7nVYA0eCpRmAhLYF5GZR5IlewCrIvjHsKOquaDBKIKOlnV9ZAzAQYeg5vTlWPNReKeLs7UM4QOhCpyrSt22yvHHWCN8QN2kzey36zJ/CpxweqAKOKZbSAJgUZwnjx/LEGv9BV6RPaNgCWD9SVg2nJ8Ab17ZGnMUT0QL6Gjl5Y4vJsnOVi6IptoAJmUTAndK+HMxoTduEoUdFv1OtRU3pxTZ178upowwbwwkGclngu6uwb/+jRgSB3qyC8Yjmi3SI1jARCag2Ah+Tg+Hdu7ucGCWdI+P2UCSHxSItiRcTIUTfoagdBoCU4CJUGTX+Pd58b3b+63tmDGrCW5SeIGqe64tutq6x6aGZ2zh2La8SlIYBdJwocYe+8CvjbD61lk=
+    #AWS_S3_REGION
+    - secure: HvWilVKzfbMuonKUBY3my7JTYxcPXSPVdeyBfrm4J42QPWFqKAk+i3yjHJWs/X7LJix6tOclYnznXBUBLbsrOGOafLoyoOtGJkkMF2th/oWti7rgsJJ0F5BMB3+v5UaD/xEBSlBRC+q6mqo+BF0VDCHybfkjQ6zR+IzInjNaVZPVDMZ/3fOIUoQ2YOZcNr6waHtzxP3UFRwifjJDfa5iNbo77BtuJ1DH6bZVExpVO18nmxhUUBDmmwAn/oJ7UL0xQhRuop0lD2yih7Xxfxby7Q9qGHrfBPaKeSlwwQ0i0mqixrrfN7Ru/aU7WJ0aPZyi1UbYraGtW4/uUUbacDqZCza7fXih3LI6J0//tPT0dg/PI8PFZuzFRQF4x5Tatbv4ePNrio9T0xKM8nDiCjOG6bVm/JwLLaLp2uvbPrWkiKbE2Nyo/rScD+tU+0xEnDU98DP8lTK60vDVviyLalYRkoP3v89LUJ9S8OocdbRm3erhvdHXqAcAOSsMwTVDvsZV9cb2Zy/4/29m+Y8PeRcSrcCuw71suAoBmjJXsQovwhSyezmDyqvDCgaqoJc9VyUNHrQKWQCJFVC+4jAlRVB2A5YcrU1vJwARkdHGQP2fcfu58U+ErXuODKlxub4ZijI8cnb9+SPbYfn38Mv8bWsjh2QkDCpVky/nUPV2NflxRjc=
+    #AWS_S3_BUCKET
+    - secure: xjnqd7FocCsBUkPKdHAwCUqLI7OQpIRn4fmJgu84JvRYdky5RqS+ZaTSv4YMqqTG3Mad0E/brEKMx4K3swErPtCzLal76Zxb9oOnkvUB5MqQ3FffftHAG26x5D0Ja7lI5QqwGrfg7lfBDcJXlIPTP86ZuRJ6uwn42vjFBOQrAESSoaHI9hHMwM6aGvQGtUPyyAPTqlu7ATgBWoT9/+MVxiuiEuoNDE40XehL2Cmu9eP5mL/A0ucG/YaUhIEzvPUYh8oF9JGariSG5ilj8DZiveN+j5lyFKMSy0S6qyjuZVSWPM7rr7a3oxrrMKbVa/nQkSa9/DnHoVa/XOZDqXhR62CE+aF0Ay0VtrGKT3NxsRpCoX/QvvWVCPKrTx6Ps4ZxTlOUTsG52OpuVdIf07rfB2tR3R7BOSs0xBl3d/FW1ZitNVe1tQu75tbwg2XF5oVVXGvyKIJaLUgP5wYgm2iGaMDYPtp7SLUJmEgvw2Ppr/VxcfI2OSyE7PhoXYzguvUKEBf4FsfivJEfIphLq5dMu6hYxHgZ3huc+duh6L+uPEetpZI9VQKaj9OvPitK9mgsGshr4twE/yl8IL998EjkxuDZmqYvv4kwj09f1TJMpQeo8tP2PaEY07Lcd+sNSE7QuUVPBbM4KFkiZXz9w1Nsl3RBVIPZbAEzmIR15C9o5SU=
+    #GOOGLE_ID
+    - secure: sx5QXSTOcklS8F0q3PvFQsyME/0S/tORYCv1SNKAPlbzVCkPuYIjM5/gUoaYyFoo+dguEO5OsaeCyIf1sAbg8xQlonuPalIxJT+hU03yTtZNPaTGgXMbbQ2R8lYzrOaKaHxPp36a5ihv96BxCW4ssBjOcPoCz/Ae2v9XPFeZDJrhMfwwoo8rDrLDj90uBpsSIK22rcBzXRpi1t36xUjE3ri2cMddQnXIrX/p8aiDcYMi+a4EqwRdtC7/VAB3a+V74uqC7vpAuhzA0YEe4U1K0V86Z0/CAYx0TD9KWt2EuImgcfgFo/8272wm+nRzK+hP44sjjal+3Y1KkvzJyBK/yszYL2niGbm4HsPHIaDBBmKODR5i7hn0HnjJ/6GHFe6X2oSHSvJVe0K7pFN0wcYIzLBdPh9jv7yX5/3oih8RuqqUKQAynmgROuiAQ8VuYu9nUjsE/Zi0B7GX5nPJ4zWtCGlblAPa+CQTYETE1g/TB7xut7GVsJJj2J8izDevypEx8l3oJsTEBxMpaKZJvoOBUnjJJOeD4gcmtiljhjQw0D+TfyRkU/JJHLJJ02YTyJfxAJ8eOCeE1wSx0lg+btcxSav/0Q4uK+L8ifF6LzFnDBl6q8OCJHe0BEppxWNFr91I/ChC/sCyR7gWmhbEj1AiENSVLZv0jl+hRPp5qKgBwNE=
+    #GOOGLE_TOKEN
+    - secure: 1yRf7EebS0XSy59q5vINFgqxEwdZKNK3/cdbWAMZ7DvZsvYkeoOrgHAzxUi+BkuLT0cnObpceQKGko47dD7If0IsKTO/N/91KVOD5vCNz/bLM6MSjf46Ien/S8MXfLfWux566IAVocdoVZghAjhwj80uSvn+Lp8LMBhnFrwf7x+f2AwLsMBhybwI+5h2WXE654zNAgnHIzud3AAURN0IFsRkXNLEd6ZH4w6Fg2SCHlCcaxd/BYQ4djUgIvRXAgCL7xSKsCnNxg/AqkkWKPQ5gu02FZLXXW0M1j+018PLJ7puRp4XQgnDK/xbD5y7mv/AGC1w2oThcZeflmY+5txd/4c4ps9B3eJdGXMLPIIg+v2XIqSjYhMUjg9y0TLQsQsih3fBrHOymZwcl06+e8eWfVQ6OkW2/7SGXdCJXwTrm8Mrx1uhmWrg/syd1sIfu5xHqxIZk51QUeXNyN0k1B42BlAznEWahif1iS+vdlD8EiKqaIRnMe1U9jHkcsXFebtirdF2i/FCSgnjXGFo1H0MXq866FDkeP+bw+5YSG6CsWEu4ovq37V8jzLBu9vyodOW40Wws03Xc7DtxjNHlYrhqOfA+x+jmokxcpKtaFHk8pBxlPxVZArTmEO3je8QgdHeM505tBfbBtprgTfZHMzJmaVqhVq65Bl3foG987eFVpw=
+
+script:
+  - ./e2e/e2e.sh
 deploy:
   provider: script
   skip_cleanup: true

blobs_reflector/reflect.go

@@ -13,7 +13,6 @@ import (
 	"github.com/lbryio/reflector.go/reflector"
 	"github.com/lbryio/reflector.go/store"
 	"github.com/lbryio/ytsync/util"
-	"github.com/mitchellh/go-ps"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -23,7 +22,7 @@ func ReflectAndClean() error {
 	if err != nil {
 		return err
 	}
-	return cleanupLbrynet()
+	return util.CleanupLbrynet()
 }
 
 func loadConfig(path string) (cmd.Config, error) {
@@ -46,7 +45,7 @@ func reflectBlobs() error {
 		return nil
 	}
 	//make sure lbrynet is off
-	running, err := isLbrynetRunning()
+	running, err := util.IsLbrynetRunning()
 	if err != nil {
 		return err
 	}
@@ -95,60 +94,3 @@ func reflectBlobs() error {
 	}
 	return nil
 }
-
-func cleanupLbrynet() error {
-	//make sure lbrynet is off
-	running, err := isLbrynetRunning()
-	if err != nil {
-		return err
-	}
-	if running {
-		return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
-	}
-	lbrynetDir := util.GetLBRYNetDir()
-	files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
-	if err != nil {
-		return errors.Err(err)
-	}
-	for _, f := range files {
-		err = os.Remove(f)
-		if err != nil {
-			return errors.Err(err)
-		}
-	}
-	blobsDir := util.GetBlobsDir()
-	err = os.RemoveAll(blobsDir)
-	if err != nil {
-		return errors.Err(err)
-	}
-	err = os.Mkdir(blobsDir, 0755)
-	if err != nil {
-		return errors.Err(err)
-	}
-	return nil
-}
-
-func isLbrynetRunning() (bool, error) {
-	if util.IsUsingDocker() {
-		container, err := util.GetLBRYNetContainer(util.ONLINE)
-		if err != nil {
-			return false, err
-		}
-		return container != nil, nil
-	}
-
-	processes, err := ps.Processes()
-	if err != nil {
-		return true, errors.Err(err)
-	}
-	var daemonProcessId = -1
-	for _, p := range processes {
-		if p.Executable() == "lbrynet" {
-			daemonProcessId = p.Pid()
-			break
-		}
-	}
-
-	running := daemonProcessId != -1
-	return running, nil
-}

e2e/docker-compose.yml

@@ -33,6 +33,8 @@ services:
       - TCP_PORT=50001
     ports:
       - "15300:50001"
+    depends_on:
+      - lbrycrd
     ulimits:
       nofile: 90000
     #    command: lbry.wallet.server.coin.LBC
@@ -46,10 +48,45 @@ services:
     ports:
       - "15100:5279"
       - "15101:5280"
+    depends_on:
+      - walletserver
     environment:
       - LBRY_STREAMING_SERVER=0.0.0.0:5280
     volumes:
       - "./persist/.lbrynet:/home/lbrynet"
       - ".:/etc/lbry" #Put your daemon_settings.yml here
       # /private/var/tmp for OSX and /var/tmp for Linux
-      - "/private/var/tmp:/var/tmp"
+      - "${LOCAL_TMP_DIR}:/var/tmp"
+    #entrypoint: wait-for-it walletserver:50001 -- start
+  ###########
+  ## MySQL ##
+  ###########
+  mysql:
+    image: mysql/mysql-server:5.7.27
+    restart: "no"
+    ports:
+      - "15500:3306"
+    environment:
+      - MYSQL_ALLOW_EMPTY_PASSWORD=true
+      - MYSQL_DATABASE=lbry
+      - MYSQL_USER=lbry
+      - MYSQL_PASSWORD=lbry
+      - MYSQL_LOG_CONSOLE=true
+  ###################
+  ## Internal APIs ##
+  ###################
+  internalapis:
+    image: lbry/internal-apis:latest
+    restart: "no"
+    ports:
+      - "15400:8080"
+    depends_on:
+      - mysql
+      - lbrycrd
+    environment:
+      - MYSQL_DSN=lbry:lbry@tcp(mysql:3306)/lbry
+      - LBRYCRD_CONNECT=rpc://lbry:lbry@lbrycrd:29245
+      - MYSQL_USER=lbry
+      - MYSQL_PASS=lbry
+      - MYSQL_DATABASE=lbry
+    entrypoint: wait-for-it mysql:3306 -- ./latest serve

e2e/e2e.sh

+#!/usr/bin/env bash
+
+set -e
+
+#Always compile ytsync
+make -C ./..
+
+#OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details
+export LOCAL_TMP_DIR="/var/tmp"
+
+#Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET
+touch -a .env && set -o allexport; source ./../.env; set +o allexport
+echo "LOCAL_TMP_DIR=$LOCAL_TMP_DIR"
+# Compose settings - docker only
+export SLACK_CHANNEL="ytsync-travis"
+export LBRY_API_TOKEN="ytsyntoken"
+export LBRY_WEB_API="http://localhost:15400"
+export LBRYNET_ADDRESS="http://localhost:15100"
+export LBRYCRD_STRING="tcp://lbry:lbry@localhost:15200"
+export LBRYNET_USE_DOCKER=true
+export REFLECT_BLOBS=false
+export CLEAN_ON_STARTUP=true
+export REGTEST=true
+# Local settings
+export BLOBS_DIRECTORY="./blobsfiles"
+export LBRYNET_DIR="`pwd`/persist/.lbrynet/.local/share/lbry/lbrynet/"
+export LBRYNET_WALLETS_DIR="`pwd`/persist/.lbrynet/.local/share/lbry/lbryum"
+export TMP_DIR="/var/tmp"
+
+docker-compose down
+rm -dr ./persist
+docker-compose up -d
+sleep 15
+
+#Data Setup for test
+#Add a ytsync user
+ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
+#Insert an auth token for the user to be used by ytsync
+ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
+#Give priveledges to ytsync user
+ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
+
+#Add youtuber to sync
+ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
+#Add their youtube channel to be synced
+ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,google_id,google_token,created_at,source,total_videos,total_subscribers)
+VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued',$GOOGLE_ID,'$GOOGLE_TOKEN','2019-08-01 00:00:00','sync',1,0)"
+mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
+
+
+# Execute the test!
+./../bin/ytsync --channelID="UCCyr5j8akeu9j4Q7urV0Lqw" #Force channel intended...just in case. This channel lines up with the api container
\ No newline at end of file

e2e/lbrycrd/docker/advance_blocks.sh

 #!/usr/bin/env bash
 while true; do
-        lbrycrd-cli -conf=/data/.lbrycrd/lbrycrd.conf generate 1 >> /tmp/output.log
+        lbrycrd-cli -conf=/etc/lbry/lbrycrd.conf generate 100 >> /tmp/output.log
         sleep 2
 done
\ No newline at end of file

e2e/lbrycrd/docker/build.sh

+#!/bin/bash
+if [ $# -eq 0 ]
+  then
+    echo "No docker tag argument supplied. Use './build.sh <tag>'"
+    exit 1
+fi
+docker build --tag lbry/lbrycrd:$1 .
+docker push lbry/lbrycrd:$1
\ No newline at end of file

e2e/lbrynet/docker/Dockerfile

 ## This base image is for running the latest lbrynet-daemon release.
 FROM ubuntu:18.04 as prep
 LABEL MAINTAINER="leopere [at] nixc [dot] us"
-RUN apt-get update && apt-get -y install unzip curl telnet
+RUN apt-get update && apt-get -y install unzip curl telnet wait-for-it
 
 ## Add lbrynet
 ARG VERSION="v0.38.5"

e2e/lbrynet/docker/build.sh

+#!/bin/bash
+if [ $# -eq 0 ]
+  then
+    echo "No docker tag argument supplied. Use './build.sh <tag>'"
+    exit 1
+fi
+docker build --tag lbry/lbrynet:$1 .
+docker push lbry/lbrynet:$1
\ No newline at end of file

manager/manager.go

@@ -101,6 +101,10 @@ const (
 
 func (s *SyncManager) Start() error {
 
+	if logUtils.IsCleanOnStartup() {
+		logUtils.CleanForStartup()
+	}
+
 	syncCount := 0
 	for {
 		err := s.checkUsedSpace()

manager/ytsync.go

 package manager
 
 import (
-	"context"
 	"fmt"
 	"io/ioutil"
 	"net/http"
 	"os"
-	"os/exec"
 	"os/signal"
 	"runtime/debug"
 	"sort"
@@ -32,8 +30,6 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/aws/aws-sdk-go/service/s3/s3manager"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/client"
 	"github.com/mitchellh/go-ps"
 	log "github.com/sirupsen/logrus"
 	"google.golang.org/api/googleapi/transport"
@@ -187,18 +183,12 @@ func (s *Sync) getWalletPaths() (defaultWallet, tempWallet string, key *string,
 }
 
 func (s *Sync) uploadWallet() error {
-	defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
+	defaultWalletDir := logUtils.GetDefaultWalletPath()
 	key := aws.String("/wallets/" + s.YoutubeChannelID)
 	if os.Getenv("REGTEST") == "true" {
-		defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
 		key = aws.String("/regtest/" + s.YoutubeChannelID)
 	}
 
-	walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
-	if walletPath != "" {
-		defaultWalletDir = walletPath + "/wallets/default_wallet"
-	}
-
 	if _, err := os.Stat(defaultWalletDir); os.IsNotExist(err) {
 		return errors.Err("default_wallet does not exist")
 	}
@@ -295,7 +285,7 @@ func (s *Sync) FullCycle() (e error) {
 
 	defer deleteSyncFolder(s.videoDirectory)
 	log.Printf("Starting daemon")
-	err = startDaemon()
+	err = logUtils.StartDaemon()
 	if err != nil {
 		return err
 	}
@@ -363,7 +353,7 @@ func (s *Sync) waitForDaemonStart() error {
 
 func (s *Sync) stopAndUploadWallet(e *error) {
 	log.Printf("Stopping daemon")
-	shutdownErr := stopDaemon()
+	shutdownErr := logUtils.StopDaemon()
 	if shutdownErr != nil {
 		logShutdownError(shutdownErr)
 	} else {
@@ -946,74 +936,6 @@ func (s *Sync) processVideo(v video) (err error) {
 	return nil
 }
 
-func startDaemon() error {
-	if logUtils.IsUsingDocker() {
-		return startDaemonViaDocker()
-	}
-	return startDaemonViaSystemd()
-}
-
-func stopDaemon() error {
-	if logUtils.IsUsingDocker() {
-		return stopDaemonViaDocker()
-	}
-	return stopDaemonViaSystemd()
-}
-
-func startDaemonViaDocker() error {
-	container, err := logUtils.GetLBRYNetContainer(true)
-	if err != nil {
-		return err
-	}
-
-	cli, err := client.NewEnvClient()
-	if err != nil {
-		panic(err)
-	}
-
-	err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
-	if err != nil {
-		return errors.Err(err)
-	}
-
-	return nil
-}
-
-func stopDaemonViaDocker() error {
-	container, err := logUtils.GetLBRYNetContainer(false)
-	if err != nil {
-		return err
-	}
-
-	cli, err := client.NewEnvClient()
-	if err != nil {
-		panic(err)
-	}
-
-	err = cli.ContainerStop(context.Background(), container.ID, nil)
-	if err != nil {
-		return errors.Err(err)
-	}
-
-	return nil
-}
-
-func startDaemonViaSystemd() error {
-	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
-	if err != nil {
-		return errors.Err(err)
-	}
-	return nil
-}
-
-func stopDaemonViaSystemd() error {
-	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
-	if err != nil {
-		return errors.Err(err)
-	}
-	return nil
-}
-
 // waitForDaemonProcess observes the running processes and returns when the process is no longer running or when the timeout is up
 func waitForDaemonProcess(timeout time.Duration) error {
 	processes, err := ps.Processes()

util/util.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"os/exec"
 	"os/user"
+	"path/filepath"
 
 	"github.com/lbryio/lbry.go/extras/errors"
 	"github.com/lbryio/lbry.go/lbrycrd"
@@ -12,6 +14,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
+	"github.com/mitchellh/go-ps"
 	log "github.com/sirupsen/logrus"
 )
 
@@ -102,3 +105,187 @@ func GetLbrycrdClient(lbrycrdString string) (*lbrycrd.Client, error) {
 
 	return lbrycrdd, nil
 }
+
+func IsCleanOnStartup() bool {
+	return os.Getenv("CLEAN_ON_STARTUP") == "true"
+}
+
+func IsLbrynetRunning() (bool, error) {
+	if IsUsingDocker() {
+		container, err := GetLBRYNetContainer(ONLINE)
+		if err != nil {
+			return false, err
+		}
+		return container != nil, nil
+	}
+
+	processes, err := ps.Processes()
+	if err != nil {
+		return true, errors.Err(err)
+	}
+	var daemonProcessId = -1
+	for _, p := range processes {
+		if p.Executable() == "lbrynet" {
+			daemonProcessId = p.Pid()
+			break
+		}
+	}
+
+	running := daemonProcessId != -1
+	return running, nil
+}
+
+func CleanForStartup() error {
+	if !IsRegTest() {
+		return errors.Err("never cleanup wallet outside of regtest and with caution. this should only be done in local testing and requires regtest to be on")
+	}
+
+	running, err := IsLbrynetRunning()
+	if err != nil {
+		return err
+	}
+	if running {
+		err := StopDaemon()
+		if err != nil {
+			return err
+		}
+	}
+
+	err = CleanupLbrynet()
+	if err != nil {
+		return err
+	}
+
+	lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
+	if err != nil {
+		return errors.Prefix("error getting lbrycrd client: ", err)
+	}
+	height, err := lbrycrd.GetBlockCount()
+	if err != nil {
+		return errors.Err(err)
+	}
+	if height < 110 {
+		//Start reg test will some credits
+		txs, err := lbrycrd.Generate(uint32(110) - uint32(height))
+		if err != nil {
+			return errors.Err(err)
+		}
+		log.Debugf("REGTEST: Generated %d transactions to get some LBC!", len(txs))
+	}
+
+	defaultWalletDir := GetDefaultWalletPath()
+	return os.Remove(defaultWalletDir)
+}
+
+func CleanupLbrynet() error {
+	//make sure lbrynet is off
+	running, err := IsLbrynetRunning()
+	if err != nil {
+		return err
+	}
+	if running {
+		return errors.Prefix("cannot cleanup lbrynet as the daemon is running", err)
+	}
+	lbrynetDir := GetLBRYNetDir()
+	files, err := filepath.Glob(lbrynetDir + "lbrynet.sqlite*")
+	if err != nil {
+		return errors.Err(err)
+	}
+	for _, f := range files {
+		err = os.Remove(f)
+		if err != nil {
+			return errors.Err(err)
+		}
+	}
+	blobsDir := GetBlobsDir()
+	err = os.RemoveAll(blobsDir)
+	if err != nil {
+		return errors.Err(err)
+	}
+	err = os.Mkdir(blobsDir, 0755)
+	if err != nil {
+		return errors.Err(err)
+	}
+	return nil
+}
+
+func StartDaemon() error {
+	if IsUsingDocker() {
+		return startDaemonViaDocker()
+	}
+	return startDaemonViaSystemd()
+}
+
+func StopDaemon() error {
+	if IsUsingDocker() {
+		return stopDaemonViaDocker()
+	}
+	return stopDaemonViaSystemd()
+}
+
+func startDaemonViaDocker() error {
+	container, err := GetLBRYNetContainer(true)
+	if err != nil {
+		return err
+	}
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		panic(err)
+	}
+
+	err = cli.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	return nil
+}
+
+func stopDaemonViaDocker() error {
+	container, err := GetLBRYNetContainer(ONLINE)
+	if err != nil {
+		return err
+	}
+
+	cli, err := client.NewEnvClient()
+	if err != nil {
+		panic(err)
+	}
+
+	err = cli.ContainerStop(context.Background(), container.ID, nil)
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	return nil
+}
+
+func startDaemonViaSystemd() error {
+	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "start", "lbrynet.service").Run()
+	if err != nil {
+		return errors.Err(err)
+	}
+	return nil
+}
+
+func stopDaemonViaSystemd() error {
+	err := exec.Command("/usr/bin/sudo", "/bin/systemctl", "stop", "lbrynet.service").Run()
+	if err != nil {
+		return errors.Err(err)
+	}
+	return nil
+}
+
+func GetDefaultWalletPath() string {
+	defaultWalletDir := os.Getenv("HOME") + "/.lbryum/wallets/default_wallet"
+	if os.Getenv("REGTEST") == "true" {
+		defaultWalletDir = os.Getenv("HOME") + "/.lbryum_regtest/wallets/default_wallet"
+	}
+
+	walletPath := os.Getenv("LBRYNET_WALLETS_DIR")
+	if walletPath != "" {
+		defaultWalletDir = walletPath + "/wallets/default_wallet"
+	}
+	return defaultWalletDir
+}