Unverified Commit b6a4eb57 authored by Mark Beamer Jr's avatar Mark Beamer Jr
Browse files

change to master tag of internal-apis docker image

parent f744e05e
Pipeline #452 failed with stage
in 1 minute and 48 seconds
......@@ -19,17 +19,23 @@ addons:
apt:
update: true
packages:
- youtube-dl
- ffmpeg
- tree
- python-pip
before_script:
- sudo pip install -U youtube-dl
env:
global:
- GO111MODULE=on
#GITHUB_TOKEN
- secure: "Ps3KocRP5xnM3/uA99CeYhDTVxRIuW7fGyrtqBeRWZW0cXzeA4XCTKxqcFbrPUPw67XkrBVgE58JDdWoQEJ7tm67PjMm/ltp5Evhx/QAJDh+YSofXyGDVpG1mrTZFI66R3NVVJLkSGALMkuWWXvfYZeU//AworJbyRoaIK/CVt5OP23i5N4tdd5UXc5dfLuYqnKRynyMmCkz9c3yEIQMXoPhG2hx7l7L2BeMJvcKmVhkSN7nQayjnrbUXGm/IRqrb88lvkyBevN5E3IB2V5IKEieIPZjbD/N0IfcnAt89Z96tgDhtIbx3ZvXm92lsvHA8buqQpG9d2AmSi6GKs64lQcnGeM5o0wER2JHWl1OSa1Nr/UAo5Xb/PM65Yt3yZE8AuMKHBmbfDSBzdkTXx58AeDzFUd3kMXD/fFjeQQWyXFlOss3ygH9SObl827Txmz9OJqZaxabs5Q3AP6m3EjKjz7zfLfrgpcxJM2WBiU1bN0ZxUgZkImy/CHk5gCZ7vhcnaLiDO4HZnzY/aRJwKYQPE5i0O2nHpIfovqkc0DFBA7U/7Cjin7e1E0UZvF3meLOxMqkfc6X7QTxqQpt2Tej6jlpdxw4CTLwGUhGkAw9IAPkUB3L0EbZ1/ksGhNvGDvUeSTq8hYdMAPmA+k9jS6653V4SQ+qBMy5++tbr5AeZQI="
script:
- ./e2e/e2e.sh
#- ./e2e/e2e.sh # Hold until we can resolve the /var/tmp issue - talk to beamer/niko
- make
deploy:
provider: script
skip_cleanup: true
......
#!/usr/bin/env bash
set -e
#Add a ytsync user
ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
#Insert an auth token for the user to be used by ytsync
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
#Give priveledges to ytsync user
ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
#Add youtuber to sync
ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
#Add their youtube channel to be synced
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,created_at,source,total_videos,total_subscribers)
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued','2019-08-01 00:00:00','sync',1,0)"
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
......@@ -26,12 +26,14 @@ services:
## Wallet Server ##
###################
walletserver:
image: lbry/wallet-server:master
image: lbry/wallet-server:v0.38.5
networks:
- e2e
restart: always
#volumes:
# - "./persist/.walletserver:/database"
environment:
- DB_DIRECTORY=/tmp/
- DB_DIRECTORY=/database
- MAX_SEND=1000000000000000000000
- DAEMON_URL=http://lbry:lbry@lbrycrd:29245
- MAX_SUBS=1000000000000
......@@ -57,7 +59,7 @@ services:
image: lbry/lbrynet:v0.38.5
networks:
- e2e
restart: "no"
restart: always
ports:
- "15100:5279"
- "15101:5280"
......@@ -72,7 +74,12 @@ services:
- "./persist/.lbrynet:/home/lbrynet"
- ".:/etc/lbry" #Put your daemon_settings.yml here
# /private/var/tmp for OSX and /var/tmp for Linux
- "${LOCAL_TMP_DIR}:/var/tmp"
- "${LOCAL_TMP_DIR}"
# - type: bind
# source: "${LOCAL_TMP_DIR}"
# target: "/tmp"
# bind:
# propagation: rshared
#entrypoint: wait-for-it walletserver:50001 -- start
###########
## MySQL ##
......@@ -96,7 +103,7 @@ services:
## Internal APIs ##
###################
internalapis:
image: lbry/internal-apis:latest
image: lbry/internal-apis:master
networks:
- e2e
restart: "no"
......
......@@ -6,7 +6,7 @@ set -e
make
#OVERRIDE this in your .env file if running from mac. Check docker-compose.yml for details
export LOCAL_TMP_DIR="/var/tmp"
export LOCAL_TMP_DIR="/var/tmp:/var/tmp"
#Private Variables Set in local installations: SLACK_TOKEN,YOUTUBE_API_KEY,AWS_S3_ID,AWS_S3_SECRET,AWS_S3_REGION,AWS_S3_BUCKET
touch -a .env && set -o allexport; source ./.env; set +o allexport
......@@ -28,21 +28,17 @@ export LBRYNET_WALLETS_DIR="$(pwd)/e2e/persist/.lbrynet/.local/share/lbry/lbryum
export TMP_DIR="/var/tmp"
export UID
#ensure that docker can be run and managed from the user
USER=$(whoami)
sudo usermod -aG docker "$USER"
cd ./e2e
docker-compose stop
docker-compose rm -f
echo "$DOCKER_PASSWORD" | docker login --username "$DOCKER_USERNAME" --password-stdin
docker-compose pull
if [[ -d persist ]]; then rm -rf persist; fi
rm -rf persist
mkdir persist
mkdir -p blobsfiles
mkdir persist/.lbrynet
chmod 777 -R ./persist
mkdir -m 0777 -p ./persist
mkdir -m 777 -p ./persist/.walletserver
mkdir -m 777 -p ./persist/.lbrynet
#sudo chown -Rv 999:999 ./persist/.walletserver
#sudo chown -Rv 1000:1000 ./persist/.lbrynet
docker-compose up -d
printf 'waiting for internal apis'
until curl --output /dev/null --silent --head --fail http://localhost:15400; do
......@@ -51,38 +47,19 @@ until curl --output /dev/null --silent --head --fail http://localhost:15400; do
done
echo "successfully started..."
# make sure we have permission to mess with the volumes
sudo find ./persist -type d -exec chmod 777 {} \;
sudo chown "$USER": -R ./persist
#Data Setup for test
#Add a ytsync user
ADDYTSYNCUSER='INSERT INTO user (given_name) VALUE("ytsync user")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCUSER"
#Insert an auth token for the user to be used by ytsync
ADDYTSYNCAUTHTOKEN='INSERT INTO auth_token (user_id, value) VALUE(1,"ytsyntoken")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCAUTHTOKEN"
#Give priveledges to ytsync user
ASSIGNGROOP='INSERT INTO user_groop (user_id, groop_id) VALUE( 1,3)'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ASSIGNGROOP"
#Add youtuber to sync
ADDYTSYNCER='INSERT INTO user (given_name) VALUE("youtuber")'
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTSYNCER"
#Add their youtube channel to be synced
ADDYTCHANNEL="INSERT INTO youtube_data (user_id, status_token,desired_lbry_channel,channel_id,channel_name,status,google_id,google_token,created_at,source,total_videos,total_subscribers)
VALUE(2,'3qzGyuVjQaf7t4pKKu2Er1NRW2LJkeWw','@beamertest','UCCyr5j8akeu9j4Q7urV0Lqw','BeamerAtLBRY','queued',$GOOGLE_ID,'$GOOGLE_TOKEN','2019-08-01 00:00:00','sync',1,0)"
mysql -u lbry -plbry -D lbry -h "127.0.0.1" -P 15500 -e "$ADDYTCHANNEL"
./data_setup.sh
# Execute the test!
./../bin/ytsync --channelID UCCyr5j8akeu9j4Q7urV0Lqw #Force channel intended...just in case. This channel lines up with the api container
# Assert the status
status=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM youtube_data WHERE id=1')
videoStatus=$(mysql -u lbry -plbry -ss -D lbry -h "127.0.0.1" -P 15500 -e 'SELECT status FROM synced_video WHERE id=1')
if [[ $status != "synced" && $videoStatus != "published" ]]; then
if [[ $status != "synced" || $videoStatus != "published" ]]; then
docker-compose logs --tail="all" lbrycrd
docker-compose logs --tail="all" walletserver
docker-compose logs --tail="all" lbrynet
docker-compose logs --tail="all" internalapis
echo "List local /var/tmp"
find /var/tmp
exit 1; fi;
\ No newline at end of file
......@@ -112,7 +112,7 @@ func (s *SyncManager) Start() error {
for {
err := s.checkUsedSpace()
if err != nil {
return err
return errors.Err(err)
}
var syncs []Sync
......@@ -122,7 +122,7 @@ func (s *SyncManager) Start() error {
if isSingleChannelSync {
channels, err := s.apiConfig.FetchChannels("", s.syncProperties)
if err != nil {
return err
return errors.Err(err)
}
if len(channels) != 1 {
return errors.Err("Expected 1 channel, %d returned", len(channels))
......
......@@ -289,6 +289,10 @@ func (s *Sync) FullCycle() (e error) {
if err != nil {
return errors.Wrap(err, 0)
}
err = os.Chmod(s.videoDirectory, 0766)
if err != nil {
return errors.Err(err)
}
defer deleteSyncFolder(s.videoDirectory)
log.Printf("Starting daemon")
......
......@@ -71,7 +71,7 @@ func (a *APIConfig) FetchChannels(status string, cp *SyncProperties) ([]YoutubeC
var response apiJobsResponse
err = json.Unmarshal(body, &response)
if err != nil {
return nil, err
return nil, errors.Err(err)
}
if response.Data == nil {
return nil, errors.Err(response.Error)
......@@ -152,7 +152,7 @@ func (a *APIConfig) SetChannelStatus(channelID string, status string, failureRea
var response apiChannelStatusResponse
err := json.Unmarshal(body, &response)
if err != nil {
return nil, nil, err
return nil, nil, errors.Err(err)
}
if !response.Error.IsNull() {
return nil, nil, errors.Err(response.Error.String)
......
......@@ -178,7 +178,7 @@ func (v *YoutubeVideo) getAbbrevDescription() string {
func (v *YoutubeVideo) download(useIPv6 bool) error {
videoPath := v.getFullPath()
err := os.Mkdir(v.videoDir(), 0750)
err := os.Mkdir(v.videoDir(), 0777)
if err != nil && !strings.Contains(err.Error(), "file exists") {
return errors.Wrap(err, 0)
}
......@@ -310,6 +310,10 @@ runcmd:
if err != nil {
return errors.Err(err)
}
err = os.Chmod(v.getFullPath(), 0777)
if err != nil {
return errors.Err(err)
}
videoSize := fi.Size()
v.size = &videoSize
return nil
......
......@@ -25,7 +25,7 @@ const thumbnailPath = "/tmp/ytsync_thumbnails/"
const ThumbnailEndpoint = "https://thumbnails.lbry.com/"
func (u *thumbnailUploader) downloadThumbnail() error {
_ = os.Mkdir(thumbnailPath, 0750)
_ = os.Mkdir(thumbnailPath, 0777)
img, err := os.Create("/tmp/ytsync_thumbnails/" + u.name)
if err != nil {
return errors.Err(err)
......
......@@ -166,7 +166,7 @@ func CleanForStartup() error {
err = CleanupLbrynet()
if err != nil {
return err
return errors.Err(err)
}
lbrycrd, err := GetLbrycrdClient(os.Getenv("LBRYCRD_STRING"))
......@@ -187,7 +187,11 @@ func CleanForStartup() error {
}
defaultWalletDir := GetDefaultWalletPath()
return os.Remove(defaultWalletDir)
_, err = os.Stat(defaultWalletDir)
if os.IsNotExist(err) {
return nil
}
return errors.Err(os.Remove(defaultWalletDir))
}
func CleanupLbrynet() error {
......@@ -215,7 +219,7 @@ func CleanupLbrynet() error {
if err != nil {
return errors.Err(err)
}
err = os.Mkdir(blobsDir, 0755)
err = os.Mkdir(blobsDir, 0777)
if err != nil {
return errors.Err(err)
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment