Change fragment duration to 10s, add per-tier CRF

.github/workflows/test.yaml

@@ -18,26 +18,23 @@ jobs:
           fetch-depth: 0
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.23"
+          go-version: "1.24"
       - uses: FedericoCarboni/setup-ffmpeg@v3
         id: setup-ffmpeg
         with:
-          ffmpeg-version: "6.1.0"
+          ffmpeg-version: release
       - run: |
           make test_prepare
       - name: Run tests
         run: go test -covermode=count -coverprofile=coverage.out ./...
-  golangci:
+  lint:
     name: lint
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-go@v5
         with:
-          go-version: "1.23"
+          go-version: "1.24"
           cache: false
       - name: golangci-lint
-        uses: golangci/golangci-lint-action@v6
-        with:
-          version: v1.63
-          args: -v
+        uses: golangci/golangci-lint-action@v7

.golangci.yml

@@ -1,36 +1,52 @@
-linters-settings:
-  errcheck:
-    check-type-assertions: true
-  gci:
-    sections:
-      - standard
-      - prefix(github.com/OdyseeTeam)
-      - prefix(github.com/lbryio)
-      - default
-    custom-order: true
-    no-lex-order: true
-  goconst:
-    min-len: 2
-    min-occurrences: 3
-
+version: "2"
 linters:
-  disable-all: true
+  default: none
   enable:
     - dupl
-    - gci
-    - gocritic
     - goconst
+    - gocritic
     - gocyclo
-    - gofmt
-    - goimports
-    - gosimple
-    - govet
     - gosec
+    - govet
     - ineffassign
     - misspell
-    - staticcheck
     - sqlclosecheck
+    - staticcheck
     - unused
-
-run:
-  timeout: 10m
+  settings:
+    errcheck:
+      check-type-assertions: true
+    goconst:
+      min-len: 2
+      min-occurrences: 3
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
+  enable:
+    - gci
+    - gofmt
+    - goimports
+  settings:
+    gci:
+      sections:
+        - standard
+        - prefix(github.com/OdyseeTeam)
+        - prefix(github.com/lbryio)
+        - default
+      custom-order: true
+      no-lex-order: true
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

Makefile

@@ -23,8 +23,14 @@ cworker_image: $(BUILD_DIR)/$(GOOS)_$(GOARCH)/transcoder
 ffmpeg_image:
 	docker buildx build -f docker/Dockerfile-ffmpeg -t odyseeteam/transcoder-ffmpeg:git --platform linux/amd64 .
 
-test_down:
+tccli_image:
+	docker buildx build -f docker/Dockerfile-tccli -t odyseeteam/transcoder-tccli:latest --platform linux/amd64 .
+
+test_clean:
 	docker-compose down
+	docker volume rm -f transcoder_minio-data
+	docker volume rm -f transcoder_redis-data
+	docker volume rm -f transcoder_db-data
 
 test_prepare:
 	make transcoder

client/client.go

@@ -4,7 +4,6 @@ import (
 	"bufio"
 	"fmt"
 	"io"
-	"math"
 	"net"
 	"net/http"
 	"net/url"
@@ -96,7 +95,7 @@ type streamLocation struct {
 func Configure() *Configuration {
 	return &Configuration{
 		remoteServer: defaultRemoteServer,
-		cacheSize:    int64(math.Pow(1024, 3)),
+		cacheSize:    1024 * 1024 * 1024,
 		itemsToPrune: 100,
 		httpClient: &http.Client{
 			CheckRedirect: func(req *http.Request, via []*http.Request) error {

client/client_test.go

@@ -21,6 +21,10 @@ import (
 	"github.com/stretchr/testify/suite"
 )
 
+// To regenerate:
+//
+//	go run ./tccli transcode "lbry://@specialoperationstest#3/fear-of-death-inspirational#a"
+//	mv fear-of-death-inspirational/*.m3u8 client/testdata/known-stream
 var streamURL = "@specialoperationstest#3/fear-of-death-inspirational#a"
 var streamSDHash = "f12fb044f5805334a473bf9a81363d89bd1cb54c4065ac05be71a599a6c51efc6c6afb257208326af304324094105774"
 
@@ -50,10 +54,10 @@ var streamFragmentCases = []struct {
 	{"v1.m3u8", 0},
 	{"v2.m3u8", 0},
 	{"v3.m3u8", 0},
-	{"v0_s000000.ts", 2_000_000},
-	{"v1_s000000.ts", 760_000},
-	{"v2_s000000.ts", 300_000},
-	{"v3_s000000.ts", 120_000},
+	{"v0_s000000.ts", 2244532},
+	{"v1_s000000.ts", 621528},
+	{"v2_s000000.ts", 221840},
+	{"v3_s000000.ts", 115620},
 }
 
 func TestClientSuite(t *testing.T) {
@@ -75,7 +79,7 @@ func (s *clientSuite) TestPlayFragment() {
 	)
 
 	// Request stream and wait until it's available.
-	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
 	wait := time.NewTicker(1000 * time.Millisecond)
 Waiting:
 	for {
@@ -102,18 +106,20 @@ Waiting:
 			sz, err := c.PlayFragment(streamURL, streamSDHash, tc.name, rr, httptest.NewRequest(http.MethodGet, "/", nil))
 			s.Require().NoError(err)
 			s.Require().Equal(http.StatusOK, rr.Result().StatusCode)
-			rbody, err := io.ReadAll(rr.Result().Body)
+			receivedBodyRaw, err := io.ReadAll(rr.Result().Body)
 			s.Require().NoError(err)
 			if tc.size > 0 {
 				// Different transcoding runs produce slightly different files.
-				s.InDelta(tc.size, len(rbody), float64(tc.size)*0.2)
-				s.EqualValues(sz, len(rbody))
+				s.InDelta(tc.size, len(receivedBodyRaw), float64(tc.size)*0.25)
+				s.EqualValues(sz, len(receivedBodyRaw))
 			} else {
-				absPath, err := filepath.Abs(filepath.Join("./testdata", "known-stream", tc.name))
+				expectedFile, err := filepath.Abs(filepath.Join("./testdata", "known-stream", tc.name))
 				s.Require().NoError(err)
-				tbody, err := os.ReadFile(absPath)
+				expectedBody, err := os.ReadFile(expectedFile)
 				s.Require().NoError(err)
-				s.Equal(strings.TrimRight(string(tbody), "\n"), strings.TrimRight(string(rbody), "\n"))
+				receivedBody := strings.TrimRight(string(receivedBodyRaw), "\n")
+				receivedBody = strings.ReplaceAll(receivedBody, ",CLOSED-CAPTIONS=NONE", "")
+				s.Equal(strings.TrimRight(string(expectedBody), "\n"), receivedBody)
 			}
 			if tc.name == MasterPlaylistName {
 				s.Equal(cacheHeaderHit, rr.Result().Header.Get(cacheHeaderName))

client/testdata/known-stream/master.m3u8

@@ -1,13 +1,14 @@
 #EXTM3U
 #EXT-X-VERSION:6
-#EXT-X-STREAM-INF:BANDWIDTH=4026000,RESOLUTION=1920x1080,CODECS="avc1.4d4028,mp4a.40.2",CLOSED-CAPTIONS=NONE
+#EXT-X-STREAM-INF:BANDWIDTH=4026000,RESOLUTION=1920x1080,CODECS="avc1.4d4028,mp4a.40.2"
 v0.m3u8
 
-#EXT-X-STREAM-INF:BANDWIDTH=2890800,RESOLUTION=1280x720,CODECS="avc1.4d401f,mp4a.40.2",CLOSED-CAPTIONS=NONE
+#EXT-X-STREAM-INF:BANDWIDTH=2890800,RESOLUTION=1280x720,CODECS="avc1.4d401f,mp4a.40.2"
 v1.m3u8
 
-#EXT-X-STREAM-INF:BANDWIDTH=655600,RESOLUTION=640x360,CODECS="avc1.4d401e,mp4a.40.2",CLOSED-CAPTIONS=NONE
+#EXT-X-STREAM-INF:BANDWIDTH=655600,RESOLUTION=640x360,CODECS="avc1.4d401e,mp4a.40.2"
 v2.m3u8
 
-#EXT-X-STREAM-INF:BANDWIDTH=215600,RESOLUTION=256x144,CODECS="avc1.4d400b,mp4a.40.2",CLOSED-CAPTIONS=NONE
+#EXT-X-STREAM-INF:BANDWIDTH=215600,RESOLUTION=256x144,CODECS="avc1.4d400b,mp4a.40.2"
 v3.m3u8
+

docker-compose.yml

@@ -17,12 +17,7 @@ services:
     depends_on:
       - redis
       - db
-    deploy:
-      restart_policy:
-        condition: on-failure
-        delay: 3s
-        max_attempts: 3
-        window: 120s
+    restart: unless-stopped
   cworker:
     image: odyseeteam/transcoder-cworker:latest
     platform: linux/amd64
@@ -34,9 +29,12 @@ services:
     # build:
     #   context: .
     #   dockerfile: docker/Dockerfile-worker
-    depends_on: ["redis"]
+    depends_on:
+      - redis
+      - minio
     volumes:
       - ${PWD}/worker.ex.yml:/app/worker.yml
+    restart: unless-stopped
   # asynqmon:
   #   image: hibiken/asynqmon:latest
   #   container_name: asynqmon
@@ -93,6 +91,7 @@ services:
       - minio-data:/data
     entrypoint: >
       /bin/sh -c "
+      sleep 3;
       /usr/bin/mc config host add myminio http://minio:9000 ody odyseetes3;
       /usr/bin/mc mb myminio/transcoded;
       /usr/bin/mc anonymous set download myminio/transcoded;

docker/Dockerfile-conductor

@@ -1,4 +1,4 @@
-FROM alpine:3
+FROM alpine:3.21
 EXPOSE 8080
 
 RUN apk add --no-cache libc6-compat

docker/Dockerfile-ffmpeg

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-FROM alpine:3
+FROM alpine:3.21
 
 WORKDIR /build
 

docker/Dockerfile-ffprobe

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1
 
 FROM odyseeteam/transcoder-ffmpeg:git AS ffmpeg
-FROM alpine:3.19
+FROM alpine:3.21
 
 COPY --from=ffmpeg /build/ffprobe /usr/local/bin/
 

docker/Dockerfile-tccli

@@ -2,7 +2,7 @@
 
 # FROM odyseeteam/transcoder-gensprite:latest AS spritegen
 
-FROM alpine:3.15 AS gather
+FROM alpine:3.21 AS gather
 
 WORKDIR /build
 
@@ -11,7 +11,7 @@ RUN tar -xf ffmpeg-git-arm64-static.tar.xz && mv ffmpeg-*-static/ffmpeg ffmpeg-*
 
 RUN chmod a+x ffmpeg ffprobe
 
-FROM alpine:3.15 AS build
+FROM alpine:3.21 AS build
 
 EXPOSE 8080
 
@@ -21,7 +21,7 @@ COPY --from=gather /build/ffmpeg /build/ffprobe /usr/local/bin/
 
 WORKDIR /app
 
-COPY ./dist/linux_arm64/tccli .
+COPY ./dist/linux_amd64/tccli .
 COPY ./conductor.ex.yml ./conductor.yml
 
-# ENTRYPOINT ["./tccli"]
+ENTRYPOINT ["/app/tccli"]

encoder/pool.go

@@ -29,5 +29,5 @@ func NewPool(encoder Encoder, parallel int) pool {
 // It works slightly different from encoder.Encode but the result should eventually be the same.
 // For how to obtain encoding progress, see poolSuite.TestEncode.
 func (p pool) Encode(in, out string) *dispatcher.Result {
-	return p.Dispatcher.Dispatch(encodeTask{in, out})
+	return p.Dispatch(encodeTask{in, out})
 }

go.mod

@@ -1,6 +1,6 @@
 module github.com/OdyseeTeam/transcoder
 
-go 1.23
+go 1.24
 
 require (
 	github.com/Pallinder/go-randomdata v1.2.0
@@ -26,11 +26,11 @@ require (
 	github.com/rubenv/sql-migrate v1.4.0
 	github.com/shopspring/decimal v1.3.1
 	github.com/spf13/viper v1.15.0
+	github.com/sqlc-dev/pqtype v0.3.0
 	github.com/stretchr/testify v1.9.0
 	github.com/tabbed/pqtype v0.1.1
 	github.com/testcontainers/testcontainers-go v0.21.0
 	github.com/valyala/fasthttp v1.36.0
-	go.etcd.io/etcd/api/v3 v3.5.6
 	go.uber.org/goleak v1.1.12
 	go.uber.org/zap v1.21.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -56,7 +56,6 @@ require (
 	github.com/containerd/containerd v1.7.13 // indirect
 	github.com/containerd/continuity v0.4.2 // indirect
 	github.com/containerd/log v0.1.0 // indirect
-	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
@@ -150,3 +149,5 @@ replace github.com/draganm/miniotest v0.1.0 => github.com/anbsky/miniotest v0.1.
 replace github.com/lbryio/lbry.go/v3 => github.com/anbsky/lbry.go/v3 v3.0.6
 
 replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
+
+replace github.com/nikooo777/lbry-blobs-downloader => github.com/anbsky/lbry-blobs-downloader v1.4.2