diff --git a/docs/content/docs/about.md b/docs/content/docs/about.md deleted file mode 100644 index 4be3d71ad..000000000 --- a/docs/content/docs/about.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -title: "About" -draft: false -menu: - main: - weight: 10 - ---- - -about \ No newline at end of file diff --git a/docs/content/docs/deployment/00-prerequisites.md b/docs/content/docs/deployment/00-prerequisites.md new file mode 100644 index 000000000..6f31754d9 --- /dev/null +++ b/docs/content/docs/deployment/00-prerequisites.md @@ -0,0 +1,25 @@ +--- +title: "Prerequisites" +draft: false +weight: 10 +--- + +## Prerequisites + +We deploy GoCast on our own hardware in VMs. Any cloud hosting provider works just as well. +You will need the following hardware configuration: + +- 1 VM for the GoCast server and database. This can be a small VM if you are not expecting a lot of users. +- At least 1 VM as an edge server. This server serves the videos to the users. Network throughput is important here. If you serve lots of users, you can spin up more of these. +- At least 1 Worker VM. This server produces the stream, transcodes the vod and much more. CPU performance is important here. As you start streaming more, you can spin up more of these. +- Optional: At least 1 NVIDIA CUDA equipped Server that transcribes streams using the Whisper LLM. +- Optional: 1 VM for monitoring (grafana, prometheus, influx...). This can be a small VM as well. + +## Storage + +GoCast produces lots of large files. They'll need to be accessed by all workers and edge servers. +Thus, you'll need a shared storage solution. We use [Ceph](https://www.ceph.com/en/). +The reliability and performance of the storage solution is critical for the performance of GoCast, setting it up and running it is not trivial. +Operating a network storage solution is out of scope for this documentation. + +For this documentation, we assume that you have some sort of high performance shared filesystem mounted to the same directory on all your servers. diff --git a/docs/content/docs/deployment/01-setup-docker.md b/docs/content/docs/deployment/01-setup-docker.md new file mode 100644 index 000000000..c7157cbd3 --- /dev/null +++ b/docs/content/docs/deployment/01-setup-docker.md @@ -0,0 +1,95 @@ +--- +title: "Setup Docker" +draft: false +weight: 20 +--- + +## Software + +Install Docker on all servers/vms: https://docs.docker.com/engine/install/ + +## Create Swarm + +On one of the servers, initialize the swarm: + +```bash +$ docker swarm init + +> Swarm initialized: current node (bvz81updecsj6wjz393c09vti) is now a manager. +> +> To add a worker to this swarm, run the following command: +> +> docker swarm join \ +> --token SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-1awxwuwd3z9j1z3puu7rcgdbx \ +> 172.17.0.2:2377 +> +> To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions. +``` + +On the other servers, join the swarm: + +```bash +$ docker swarm join \ + --token SWMTKN-1-3pu6hszjas19xyp7ghgosyx9k8atbfcr8p2is99znpy26u2lkl-1awxwuwd3z9j1z3puu7rcgdbx \ + 172.17.0.2:2377 +``` + +Read the administration guide for docker swarm carefully and make the appropriate adjustments for your environment: +https://docs.docker.com/engine/swarm/admin_guide/ + +Verify that all nodes are in the swarm: + +```bash +$ docker node ls +ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION +ko66mqj76xo9ftunxq78luc8p vm01 Ready Active Reachable 23.0.1 +ogziph0qxfeivly5fnekepwx0 vm02 Ready Active 23.0.1 +1prl8b1m7xw2ph5b8dnh98glk vm03 Ready Active 23.0.1 +8utl07361ocn5xvzqh27z0c8s * vm04 Ready Active Reachable 23.0.1 +hdsuhlwecidor7khbcfn4gni3 vm05 Ready Active Reachable 23.0.1 +hj6fkl3j5hwho40uiehc7ikq5 vm06 Ready Active Leader 23.0.1 +ctfdd9mtkse2yxid8zku2wx1f vm07 Ready Active 23.0.1 +u391iukj6nljosaaygcfkzy2s vm08 Ready Active 23.0.1 +wkxct5tvzclvc4uqm8w573dlf vm09 Ready Active 23.0.1 +72weo6nozra1cdgjs5wghe7gh vm10 Ready Active 23.0.1 +``` + +## Tag nodes + +We use labels to tag our nodes and to deploy services to appropriate nodes. + +This commands adds the label worker to the node vm02, instructing our deployment to deploy workers on this node: + +```bash +docker node update --label-add worker=true vm02 +``` +This is a configuration you should aim for: + +```bash +docker node ls -q | xargs docker node inspect -f '{{ .ID }} [{{ .Description.Hostname }}]: {{ .Spec.Labels }}' + +kwgmm6sxb9nqwojoclxuy4mpt [vmgpu01]: map[voiceservice:true] # optional, this is a server with a GPU for transcription +ko66mqj76xo9ftunxq78luc8p [vm01]: map[db:true traefik:true tumlive:true] # this server is important, it runs the database and the reverse proxy. Don't under-provision it. +hj6fkl3j5hwho40uiehc7ikq5 [vm02]: map[grafana:true influx:true meilisearch:true monitoring:true prometheus:true] # these services are not critical - and optional +ctfdd9mtkse2yxid8zku2wx1f [vm03]: map[worker:true] # the number of workers depends on the number of concurrent streams you want to process. 1 worker can process around 5 stream in our environment. +u391iukj6nljosaaygcfkzy2s [vm04]: map[worker:true] +wkxct5tvzclvc4uqm8w573dlf [vm05]: map[worker:true] +72weo6nozra1cdgjs5wghe7gh [vm06]: map[worker:true] +f7ik66qq6tzhsbwphfpdp2vm1 [vm07]: map[worker:true] +i4l8ouumms96qu96evkb6srol [vm08]: map[worker:true] +vq5cw2bgwncenr5cp89xzsi32 [vm09]: map[worker:true] +q4as4i27z2hnwypgzj8ql2dz1 [vm10]: map[worker:true] +lfged5ra1a7z9wlstxa2bml5c [vm11]: map[worker:true] +3wu812ybzynnunrpoqdsay0bf [vm12]: map[worker:true] +itdbo77gempnl251lakioe5y1 [vm13]: map[worker:true] +zcplsihexr88plf0t8q25tdn7 [vm14]: map[worker:true] +fbi92hp7s0u3c2x13tgrb6fd6 [vm15]: map[worker:true] +o6k2egpupik3qjgq2w0azv70o [vm16]: map[worker:true] +urac70xjf1kx5op39kyulykad [vm17]: map[worker:true] +wpue8f384h7z71mngov5j72c1 [vm18]: map[worker:true] +th77fn3s91s06sy4ciprita3s [vm19]: map[edge:true] # the number of edge nodes depends on the number of concurrent viewers you want to support. +5bqr01nyefxqmkd3luzhh3sne [vm20]: map[edge:true] +vrroo1k8kgk8n557pos5wlz5k [vm21]: map[edge:true] +b6m40kbtg1sctwq5p4vmtghxd [vm22]: map[edge:true] + +``` diff --git a/docs/content/docs/deployment/02-networking.md b/docs/content/docs/deployment/02-networking.md new file mode 100644 index 000000000..640a9a464 --- /dev/null +++ b/docs/content/docs/deployment/02-networking.md @@ -0,0 +1,18 @@ +--- +title: "Networking" +draft: false +weight: 30 +--- + + +## Networking + +The following ports need to be exposed to the public: + +| Server (label) | Port | +|----------------------------------|-----------------| +| GoCast Server (tumlive, traefik) | 80 TCP, 443 TCP | +| Worker (worker) | 1935 TCP | +| Edge (edge) | 80 TCP, 443 TCP | + +Between the individual servers, communication should not be firewalled. Auditorium hardware should also be in the same vlan. diff --git a/docs/content/docs/deployment/03-deploy.md b/docs/content/docs/deployment/03-deploy.md new file mode 100644 index 000000000..195f72ed5 --- /dev/null +++ b/docs/content/docs/deployment/03-deploy.md @@ -0,0 +1,465 @@ +--- +title: "Deploy" +draft: false +weight: 40 +--- + +## Configuration + +To run GoCast, copy the contents of the `/docs/static/deployment` directory to your server into a shared location that is available to all nodes. +Edit the `docker-compose.yml` file to match your environment (domains, file locations,...). This is a demo of the file: + +```yaml +version: '3.8' +services: + tumlive: + image: ghcr.io/joschahenningsen/tum-live/tum-live:latest + depends_on: + - tumlivedb + ports: + - target: 50052 + published: 50052 + protocol: tcp + mode: host + volumes: + - /share/deployment/config.yaml:/etc/TUM-Live/config.yaml # todo make sure /share is available on all nodes + - /share/branding:/etc/TUM-Live/branding + - /share:/mass + - /var/lib/rbg-cert/live:/var/lib/rbg-cert/live + - /path/to/mail.p12.crt.pem:/path/to/mail.p12.crt.pem # todo change this to your mail cert + - /path/to/mail.p12.key.pem:/path/to/mail.p12.key.pem + - /var/www/public:/var/www/public + networks: + - default + environment: + - TMPDIR=/tmp + - TMP=/tmp + - GIN_MODE=release + - TZ=Europe/Berlin + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - "node.labels.tumlive==true" + labels: + - "traefik.enable=true" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.scheme=https" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.permanent=true" + + # dynamic + - "traefik.http.routers.tumlive.entrypoints=web" + - "traefik.http.routers.tumlive.rule=Host(`live.rbg.tum.de`) || Host(`tum.live`)" # todo change url/s + - "traefik.http.routers.tumlive.middlewares=webs-redirectscheme" + - "traefik.http.routers.tumlive.service=tumlive-secure" + + + - "traefik.http.routers.tumlive-secure-static.entrypoints=webs" + - "traefik.http.routers.tumlive-secure-static.tls=true" + - "traefik.http.routers.tumlive-secure-static.tls.certresolver=liveresolver" + - "traefik.http.routers.tumlive-secure-static.rule=(Host(`live.rbg.tum.de`) || Host(`tum.live`)) && PathPrefix(`/static/`, `/public/`)" # todo change url/s + - "traefik.http.routers.tumlive-secure-static.service=tumlive-secure" + - "traefik.http.services.tumlive-secure-static.loadbalancer.server.port=8081" + - "traefik.http.routers.tumlive-secure-static.middlewares=cache-headers" + + + - "traefik.http.routers.tumlive-secure.entrypoints=webs" + - "traefik.http.routers.tumlive-secure.tls=true" + - "traefik.http.routers.tumlive-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.tumlive-secure.rule=Host(`live.rbg.tum.de`) || Host(`tum.live`)" # todo change url/s + - "traefik.http.routers.tumlive-secure.service=tumlive-secure" + - "traefik.http.services.tumlive-secure.loadbalancer.server.port=8081" + + tumlivedb: + image: mariadb:latest + environment: + - MARIADB_USER=root + - MARIADB_ROOT_PASSWORD=abc123 # todo change this here and in gocasts config.yaml + - TZ=Europe/Berlin + networks: + - default + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - "node.labels.db==true" + volumes: + - mariadb_data:/var/lib/mysql + + worker: + image: ghcr.io/joschahenningsen/tum-live/worker:latest + networks: + - default + environment: + - Token=abc123 # todo change this here and in gocasts config.yaml + - MainBase=tumlive + - Host={{.Node.Hostname}} + - LrzUploadUrl=http://vodservice:8089 + - LogLevel=debug + - PersistDir=/persist + - VodURLTemplate=https://edge.live.rbg.tum.de/vod/%s.mp4/playlist.m3u8 # todo change this depending on your edge server url + ports: + - target: 1935 + published: 1935 + mode: host + protocol: tcp + volumes: + - recordings:/recordings + - persist:/persist + - /share:/mass # todo make sure /share is available on all nodes + - workerlog:/var/log/stream + deploy: + mode: global # replicate to every node + placement: + constraints: + - "node.labels.worker==true" + restart_policy: + condition: on-failure + + # optional + voice-service: + image: ghcr.io/tum-dev/tum-live-voice-service-nvidia:0.0.5 + volumes: + - /share:/mass # todo make sure /share is available on all nodes + networks: + - default + deploy: + resources: + reservations: + generic_resources: + - discrete_resource_spec: + kind: 'gpu' + value: 0 + mode: global + placement: + constraints: + - "node.labels.voiceservice==true" + environment: + - TRANSCRIBER=whisper + - WHISPER_MODEL=medium + - MAX_WORKERS=1 + - DEBUG=1 + - REC_HOST=tumlive + - REC_PORT=50053 + + edge: + image: ghcr.io/joschahenningsen/tum-live/worker-edge:latest + networks: + - default + ports: + - target: 8089 + published: 80 + mode: host + protocol: tcp + - target: 8443 + published: 443 + mode: host + protocol: tcp + environment: + - CERT_DIR=/var/lib/rbg-cert/live/ # todo, this directory must exist on all edge nodes and contain ssl certificates valid for the domain the nodes use. + - VOD_DIR=/vod + - MAIN_INSTANCE=http://tumlive:8081 + - ADMIN_TOKEN=abc123 # todo changeme + volumes: + - /share/vod:/vod # todo make sure /share is available on all hosts + - /var/lib/rbg-cert/:/var/lib/rbg-cert/ + deploy: + mode: global + endpoint_mode: dnsrr + placement: + constraints: + - "node.labels.edge==true" + + vodservice: + image: ghcr.io/joschahenningsen/tum-live/vod-service:latest + networks: + - default + ports: + # web + - target: 8089 + published: 8089 + protocol: tcp + mode: host + environment: + - OUTPUT_DIR=/out + volumes: + - /share/vod:/out # todo make sure /share is available on all hosts + deploy: + mode: global + placement: + constraints: + - "node.labels.worker==true" + + meilisearch: + image: getmeili/meilisearch:v0.30 + volumes: + - meilisearch:/meili_data + networks: + - default + environment: + - MEILI_MASTER_KEY=abc123 # todo change me + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - "node.labels.meilisearch==true" + + grafana: + image: grafana/grafana + volumes: + - grafana:/var/lib/grafana + - /shared/deployment/grafana.ini:/etc/grafana/grafana.ini # todo make sure /shared is available on all hosts + networks: + - default + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - "node.labels.grafana==true" + labels: + - "traefik.enable=true" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.scheme=https" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.permanent=true" + + # dynamic + - "traefik.http.routers.grafana.entrypoints=web" + - "traefik.http.routers.grafana.rule=Host(`grafana.my.domain`)" # todo pick a domain + - "traefik.http.routers.grafana.middlewares=webs-redirectscheme" + - "traefik.http.routers.grafana.service=grafana-secure" + + - "traefik.http.routers.grafana-secure.entrypoints=webs" + - "traefik.http.routers.grafana-secure.tls=true" + - "traefik.http.routers.grafana-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.grafana-secure.rule=Host(`grafana.my.domain`)" # todo pick a domain + - "traefik.http.routers.grafana-secure.service=grafana-secure" + - "traefik.http.services.grafana-secure.loadbalancer.server.port=3000" + + prometheus: + image: prom/prometheus + volumes: + - /shared/deployment/prometheus.yml:/etc/prometheus/prometheus.yml # todo, /shared has to be available on all hosts + - prometheus:/prometheus + networks: + - default + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - "node.labels.prometheus==true" + + traefik: + image: traefik:v2.9 + networks: + - default + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - "node.labels.traefik==true" + labels: + - "traefik.enable=true" + #General purpose redirect middleware used throughout + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.scheme=https" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.permanent=true" + - "traefik.http.middlewares.cache-headers.headers.customresponseheaders.Cache-Control=public,max-age=2592000" + + - "traefik.http.routers.traefik.entrypoints=web" + - "traefik.http.routers.traefik.rule=Host(`traefik.my.domain`)" # todo in case you need the traefik interface + - "traefik.http.routers.traefik.middlewares=webs-redirectscheme" + - "traefik.http.routers.traefik.service=traefik-secure" + + - "traefik.http.routers.traefik-secure.entrypoints=webs" + - "traefik.http.routers.traefik-secure.tls=true" + - "traefik.http.routers.traefik-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.traefik-secure.rule=Host(`traefik.my.domain`)" # todo in case you need the traefik interface + - "traefik.http.routers.traefik-secure.service=traefik-secure" + + - "traefik.http.services.traefik-secure.loadbalancer.server.port=8080" + ports: + # web + - target: 80 + published: 80 + protocol: tcp + mode: host + - target: 443 + published: 443 + protocol: tcp + mode: host + volumes: + # So that Traefik can listen to the Docker events + - /var/run/docker.sock:/var/run/docker.sock + - /srv/cephfs/livestream/TUM-Live/deployment/traefik.toml:/etc/traefik/traefik.toml:ro + - /srv/cephfs/livestream/TUM-Live/deployment/acme:/acme + - /var/log/traefik:/var/log/traefik + + whoami: + # A container that exposes an API to show its IP address + image: traefik/whoami + networks: + - default + deploy: + mode: global + + campus-proxy: + image: ghcr.io/tum-dev/campusproxy/proxy:latest + networks: + - default + deploy: + mode: replicated + replicas: 2 + labels: + - "traefik.enable=true" + - "traefik.http.routers.proxy.entrypoints=web" + - "traefik.http.routers.proxy.rule=Host(`campus-proxy.my.domain`)" # todo pick a url + - "traefik.http.routers.proxy.middlewares=webs-redirectscheme" + - "traefik.http.routers.proxy-service=proxy-secure" + + - "traefik.http.routers.proxy-secure.entrypoints=webs" + - "traefik.http.routers.proxy-secure.tls=true" + - "traefik.http.routers.proxy-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.proxy-secure.rule=Host(`campus-proxy.my.domain`)" # todo pick a url + - "traefik.http.routers.proxy-secure.service=proxy-secure" + + - "traefik.http.services.proxy-secure.loadbalancer.server.port=8020" + googleSiteVerification: + image: nginx:latest + volumes: + # todo: use your own google verification file + - /srv/cephfs/livestream/TUM-Live/deployment/google695ffe94aec91c5d.html:/usr/share/nginx/html/google695ffe94aec91c5d.html + deploy: + mode: replicated + replicas: 2 + labels: + - "traefik.enable=true" + + - "traefik.http.routers.gsv-secure.entrypoints=webs" + - "traefik.http.routers.gsv-secure.tls=true" + - "traefik.http.routers.gsv-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.gsv-secure.rule=Host(`live.rbg.tum.de`) && Path(`/google695ffe94aec91c5d.html`)" + - "traefik.http.routers.gsv-secure.service=gsv-secure" + + - "traefik.http.services.gsv-secure.loadbalancer.server.port=80" + +volumes: + recordings: + persist: + mariadb_data: + workerlog: + meilisearch: + grafana: + prometheus: + +networks: + agent_network: + driver: overlay + attachable: true + default: + driver: overlay + host: + name: host + external: true +``` + +Edit the `traefik.toml` file to your needs. You can find an example in the `deployment` folder. + +Edit the `config.yaml` file to your needs: + +```yaml +alerts: + matrix: + homeserver: matrix.org # todo changeme + password: password # todo changeme + alertRoomID: '!abc:in.tum.de' # todo changeme + logRoomID: '!abc123:matrix.org' # todo changeme + username: username # todo changeme +auths: + camauth: user:password # todo changeme + pwrcrtlauth: user:password # todo changeme + smppassword: "password" # todo changeme + smpuser: user # todo changeme +campus: + base: https://campus.tum.de/tumonlinej/ws/webservice_v1.0 # todo changeme + tokens: + - abc123 # todo changeme + campusProxy: # new services use this proxy from now on + host: campus-proxy.my.domain # todo changeme + scheme: https + relevantOrgs: # 0 = all + - 51897 # cit + - 30361 # studentische vertretung + - 30290 # fachschaften + - 14189 # institut für informatik + - 14178 # fakultät für mathematik + - 14179 # fakultät für physik + - 51267 # tum school of engineering and design + - 51900 # tum school of management +db: + database: tumlive + password: abc123 # todo changeme + user: root + host: tumlivedb + port: 3306 +ingestbase: rtmp://vmrbg458.in.tum.de/ +jwtkey: +ldap: + useForLogin: true + basedn: ou=users,ou=data,ou=prod,ou=iauth,dc=tum,dc=de # todo changeme + password: abc123 # todo changeme + url: ldaps://iauth.tum.de:636 + user: cn=usernameChangeme,ou=bindDNs,ou=iauth,dc=tum,dc=de # todo changeme + userdn: cn=%s,ou=users,ou=data,ou=prod,ou=iauth,dc=tum,dc=de # todo changeme +saml: # todo changeme + idpMetadataURL: https://login.tum.de/idp-metadata.xml + idpName: TUM Login + idpColor: "#3070B3" + cert: /var/lib/cert/live/host:intum:vmrbg451.fullchain.pem + privkey: /var/lib/cert/live/host:intum:vmrbg451.privkey.pem + entityID: https://live.rbg.tum.de/shib + rootURLs: + - https://live.rbg.tum.de/shib + - https://tum.live/shib +mail: + sender: live@my.domain # todo changeme + server: mailrelay.my.domain:25 # todo changeme + smimecert: /path/to/mail.p12.crt.pem + smimekey: /path/to/mail.p12.key.pem +paths: + mass: /share + static: /var/www/public + branding: /etc/TUM-Live/branding +workertoken: abc123 # todo changeme +weburl: https://live.rbg.tum.de +monitoring: + sentryDSN: https://abc@sentry.com/2 # todo changeme + sampleRate: 0.1 +meili: + host: http://meilisearch:7700 + apiKey: abc123 # todo changeme +vodURLTemplate: "https://edge.live.rbg.tum.de/vod/%s.mp4/playlist.m3u8" +voiceservice: + host: voice-service + port: 50055 +canonicalURL: https://live.rbg.tum.de +``` + +## Deployment + + +```bash +$ docker stack deploy -c docker-compose.yml gocast +``` + +After a few minutes, everything should be up and running, certificates are issued automatically \ No newline at end of file diff --git a/docs/content/docs/deployment/_index.md b/docs/content/docs/deployment/_index.md new file mode 100644 index 000000000..6bb1e7df2 --- /dev/null +++ b/docs/content/docs/deployment/_index.md @@ -0,0 +1,9 @@ +--- +title: "Deployment" +draft: false +bookCollapseSection: true +class: "deployment" +menu: + main: + weight: 30 +--- diff --git a/docs/content/docs/features/_index.md b/docs/content/docs/features/_index.md index c2c669b12..c3b277c97 100644 --- a/docs/content/docs/features/_index.md +++ b/docs/content/docs/features/_index.md @@ -3,8 +3,7 @@ title: "Features" draft: false bookCollapseSection: true class: "features" -menu: - main: - weight: 20 +weight: 30 + --- diff --git a/docs/content/docs/usage/01-self-streaming.md b/docs/content/docs/usage/01-self-streaming.md new file mode 100644 index 000000000..f4834cec1 --- /dev/null +++ b/docs/content/docs/usage/01-self-streaming.md @@ -0,0 +1,22 @@ +--- +title: "Self Streaming" +draft: false +weight: 10 +--- + +## Self Streaming + +This guide contains information on how to stream from your own computer. + + +### Gather streaming key. + +- Open the courses admin page +- Create a stream with the location "self stream" if it doesn't exist already. +- Navigate to the stream and click `show keys`. + +#### OBS + +#### Zoom + +To use zoom for streaming, login to your account, navigate to... Click... \ No newline at end of file diff --git a/docs/content/docs/usage/_index.md b/docs/content/docs/usage/_index.md new file mode 100644 index 000000000..6215a241e --- /dev/null +++ b/docs/content/docs/usage/_index.md @@ -0,0 +1,9 @@ +--- +title: "Usage" +draft: false +bookCollapseSection: true +class: "usage" +menu: + main: + weight: 40 +--- diff --git a/docs/static/deployment/config.yaml b/docs/static/deployment/config.yaml new file mode 100644 index 000000000..ba89e1067 --- /dev/null +++ b/docs/static/deployment/config.yaml @@ -0,0 +1,75 @@ +alerts: + matrix: + homeserver: matrix.org # todo changeme + password: password # todo changeme + alertRoomID: '!abc:in.tum.de' # todo changeme + logRoomID: '!abc123:matrix.org' # todo changeme + username: username # todo changeme +auths: + camauth: user:password # todo changeme + pwrcrtlauth: user:password # todo changeme + smppassword: "password" # todo changeme + smpuser: user # todo changeme +campus: + base: https://campus.tum.de/tumonlinej/ws/webservice_v1.0 # todo changeme + tokens: + - abc123 # todo changeme + campusProxy: # new services use this proxy from now on + host: campus-proxy.my.domain # todo changeme + scheme: https + relevantOrgs: # 0 = all + - 51897 # cit + - 30361 # studentische vertretung + - 30290 # fachschaften + - 14189 # institut für informatik + - 14178 # fakultät für mathematik + - 14179 # fakultät für physik + - 51267 # tum school of engineering and design + - 51900 # tum school of management +db: + database: tumlive + password: abc123 # todo changeme + user: root + host: tumlivedb + port: 3306 +ingestbase: rtmp://vmrbg458.in.tum.de/ +jwtkey: +ldap: + useForLogin: true + basedn: ou=users,ou=data,ou=prod,ou=iauth,dc=tum,dc=de # todo changeme + password: abc123 # todo changeme + url: ldaps://iauth.tum.de:636 + user: cn=usernameChangeme,ou=bindDNs,ou=iauth,dc=tum,dc=de # todo changeme + userdn: cn=%s,ou=users,ou=data,ou=prod,ou=iauth,dc=tum,dc=de # todo changeme +saml: # todo changeme + idpMetadataURL: https://login.tum.de/idp-metadata.xml + idpName: TUM Login + idpColor: "#3070B3" + cert: /var/lib/cert/live/host:intum:vmrbg451.fullchain.pem + privkey: /var/lib/cert/live/host:intum:vmrbg451.privkey.pem + entityID: https://live.rbg.tum.de/shib + rootURLs: + - https://live.rbg.tum.de/shib + - https://tum.live/shib +mail: + sender: live@my.domain # todo changeme + server: mailrelay.my.domain:25 # todo changeme + smimecert: /path/to/mail.p12.crt.pem + smimekey: /path/to/mail.p12.key.pem +paths: + mass: /share + static: /var/www/public + branding: /etc/TUM-Live/branding +workertoken: abc123 # todo changeme +weburl: https://live.rbg.tum.de +monitoring: + sentryDSN: https://abc@sentry.com/2 # todo changeme + sampleRate: 0.1 +meili: + host: http://meilisearch:7700 + apiKey: abc123 # todo changeme +vodURLTemplate: "https://edge.live.rbg.tum.de/vod/%s.mp4/playlist.m3u8" +voiceservice: + host: voice-service + port: 50055 +canonicalURL: https://live.rbg.tum.de diff --git a/docs/static/deployment/docker-compose.yml b/docs/static/deployment/docker-compose.yml new file mode 100644 index 000000000..332794f70 --- /dev/null +++ b/docs/static/deployment/docker-compose.yml @@ -0,0 +1,361 @@ +version: '3.8' +services: + tumlive: + image: ghcr.io/joschahenningsen/tum-live/tum-live:latest + depends_on: + - tumlivedb + ports: + - target: 50052 + published: 50052 + protocol: tcp + mode: host + volumes: + - /share/deployment/config.yaml:/etc/TUM-Live/config.yaml # todo make sure /share is available on all nodes + - /share/branding:/etc/TUM-Live/branding + - /share:/mass + - /var/lib/rbg-cert/live:/var/lib/rbg-cert/live + - /path/to/mail.p12.crt.pem:/path/to/mail.p12.crt.pem # todo change this to your mail cert + - /path/to/mail.p12.key.pem:/path/to/mail.p12.key.pem + - /var/www/public:/var/www/public + networks: + - default + environment: + - TMPDIR=/tmp + - TMP=/tmp + - GIN_MODE=release + - TZ=Europe/Berlin + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - "node.labels.tumlive==true" + labels: + - "traefik.enable=true" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.scheme=https" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.permanent=true" + + # dynamic + - "traefik.http.routers.tumlive.entrypoints=web" + - "traefik.http.routers.tumlive.rule=Host(`live.rbg.tum.de`) || Host(`tum.live`)" # todo change url/s + - "traefik.http.routers.tumlive.middlewares=webs-redirectscheme" + - "traefik.http.routers.tumlive.service=tumlive-secure" + + + - "traefik.http.routers.tumlive-secure-static.entrypoints=webs" + - "traefik.http.routers.tumlive-secure-static.tls=true" + - "traefik.http.routers.tumlive-secure-static.tls.certresolver=liveresolver" + - "traefik.http.routers.tumlive-secure-static.rule=(Host(`live.rbg.tum.de`) || Host(`tum.live`)) && PathPrefix(`/static/`, `/public/`)" # todo change url/s + - "traefik.http.routers.tumlive-secure-static.service=tumlive-secure" + - "traefik.http.services.tumlive-secure-static.loadbalancer.server.port=8081" + - "traefik.http.routers.tumlive-secure-static.middlewares=cache-headers" + + + - "traefik.http.routers.tumlive-secure.entrypoints=webs" + - "traefik.http.routers.tumlive-secure.tls=true" + - "traefik.http.routers.tumlive-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.tumlive-secure.rule=Host(`live.rbg.tum.de`) || Host(`tum.live`)" # todo change url/s + - "traefik.http.routers.tumlive-secure.service=tumlive-secure" + - "traefik.http.services.tumlive-secure.loadbalancer.server.port=8081" + + tumlivedb: + image: mariadb:latest + environment: + - MARIADB_USER=root + - MARIADB_ROOT_PASSWORD=abc123 # todo change this here and in gocasts config.yaml + - TZ=Europe/Berlin + networks: + - default + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - "node.labels.db==true" + volumes: + - mariadb_data:/var/lib/mysql + + worker: + image: ghcr.io/joschahenningsen/tum-live/worker:latest + networks: + - default + environment: + - Token=abc123 # todo change this here and in gocasts config.yaml + - MainBase=tumlive + - Host={{.Node.Hostname}} + - LrzUploadUrl=http://vodservice:8089 + - LogLevel=debug + - PersistDir=/persist + - VodURLTemplate=https://edge.live.rbg.tum.de/vod/%s.mp4/playlist.m3u8 # todo change this depending on your edge server url + ports: + - target: 1935 + published: 1935 + mode: host + protocol: tcp + volumes: + - recordings:/recordings + - persist:/persist + - /share:/mass # todo make sure /share is available on all nodes + - workerlog:/var/log/stream + deploy: + mode: global # replicate to every node + placement: + constraints: + - "node.labels.worker==true" + restart_policy: + condition: on-failure + + # optional + voice-service: + image: ghcr.io/tum-dev/tum-live-voice-service-nvidia:0.0.5 + volumes: + - /share:/mass # todo make sure /share is available on all nodes + networks: + - default + deploy: + resources: + reservations: + generic_resources: + - discrete_resource_spec: + kind: 'gpu' + value: 0 + mode: global + placement: + constraints: + - "node.labels.voiceservice==true" + environment: + - TRANSCRIBER=whisper + - WHISPER_MODEL=medium + - MAX_WORKERS=1 + - DEBUG=1 + - REC_HOST=tumlive + - REC_PORT=50053 + + edge: + image: ghcr.io/joschahenningsen/tum-live/worker-edge:latest + networks: + - default + ports: + - target: 8089 + published: 80 + mode: host + protocol: tcp + - target: 8443 + published: 443 + mode: host + protocol: tcp + environment: + - CERT_DIR=/var/lib/rbg-cert/live/ # todo, this directory must exist on all edge nodes and contain ssl certificates valid for the domain the nodes use. + - VOD_DIR=/vod + - MAIN_INSTANCE=http://tumlive:8081 + - ADMIN_TOKEN=abc123 # todo changeme + volumes: + - /share/vod:/vod # todo make sure /share is available on all hosts + - /var/lib/rbg-cert/:/var/lib/rbg-cert/ + deploy: + mode: global + endpoint_mode: dnsrr + placement: + constraints: + - "node.labels.edge==true" + + vodservice: + image: ghcr.io/joschahenningsen/tum-live/vod-service:latest + networks: + - default + ports: + # web + - target: 8089 + published: 8089 + protocol: tcp + mode: host + environment: + - OUTPUT_DIR=/out + volumes: + - /share/vod:/out # todo make sure /share is available on all hosts + deploy: + mode: global + placement: + constraints: + - "node.labels.worker==true" + + meilisearch: + image: getmeili/meilisearch:v0.30 + volumes: + - meilisearch:/meili_data + networks: + - default + environment: + - MEILI_MASTER_KEY=abc123 # todo change me + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - "node.labels.meilisearch==true" + + grafana: + image: grafana/grafana + volumes: + - grafana:/var/lib/grafana + - /shared/deployment/grafana.ini:/etc/grafana/grafana.ini # todo make sure /shared is available on all hosts + networks: + - default + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - "node.labels.grafana==true" + labels: + - "traefik.enable=true" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.scheme=https" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.permanent=true" + + # dynamic + - "traefik.http.routers.grafana.entrypoints=web" + - "traefik.http.routers.grafana.rule=Host(`grafana.my.domain`)" # todo pick a domain + - "traefik.http.routers.grafana.middlewares=webs-redirectscheme" + - "traefik.http.routers.grafana.service=grafana-secure" + + - "traefik.http.routers.grafana-secure.entrypoints=webs" + - "traefik.http.routers.grafana-secure.tls=true" + - "traefik.http.routers.grafana-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.grafana-secure.rule=Host(`grafana.my.domain`)" # todo pick a domain + - "traefik.http.routers.grafana-secure.service=grafana-secure" + - "traefik.http.services.grafana-secure.loadbalancer.server.port=3000" + + prometheus: + image: prom/prometheus + volumes: + - /shared/deployment/prometheus.yml:/etc/prometheus/prometheus.yml # todo, /shared has to be available on all hosts + - prometheus:/prometheus + networks: + - default + deploy: + mode: replicated + replicas: 1 + restart_policy: + condition: on-failure + placement: + constraints: + - "node.labels.prometheus==true" + + traefik: + image: traefik:v2.9 + networks: + - default + deploy: + mode: replicated + replicas: 1 + placement: + constraints: + - "node.labels.traefik==true" + labels: + - "traefik.enable=true" + #General purpose redirect middleware used throughout + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.scheme=https" + - "traefik.http.middlewares.webs-redirectscheme.redirectscheme.permanent=true" + - "traefik.http.middlewares.cache-headers.headers.customresponseheaders.Cache-Control=public,max-age=2592000" + + - "traefik.http.routers.traefik.entrypoints=web" + - "traefik.http.routers.traefik.rule=Host(`traefik.my.domain`)" # todo in case you need the traefik interface + - "traefik.http.routers.traefik.middlewares=webs-redirectscheme" + - "traefik.http.routers.traefik.service=traefik-secure" + + - "traefik.http.routers.traefik-secure.entrypoints=webs" + - "traefik.http.routers.traefik-secure.tls=true" + - "traefik.http.routers.traefik-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.traefik-secure.rule=Host(`traefik.my.domain`)" # todo in case you need the traefik interface + - "traefik.http.routers.traefik-secure.service=traefik-secure" + + - "traefik.http.services.traefik-secure.loadbalancer.server.port=8080" + ports: + # web + - target: 80 + published: 80 + protocol: tcp + mode: host + - target: 443 + published: 443 + protocol: tcp + mode: host + volumes: + # So that Traefik can listen to the Docker events + - /var/run/docker.sock:/var/run/docker.sock + - /srv/cephfs/livestream/TUM-Live/deployment/traefik.toml:/etc/traefik/traefik.toml:ro + - /srv/cephfs/livestream/TUM-Live/deployment/acme:/acme + - /var/log/traefik:/var/log/traefik + + whoami: + # A container that exposes an API to show its IP address + image: traefik/whoami + networks: + - default + deploy: + mode: global + + campus-proxy: + image: ghcr.io/tum-dev/campusproxy/proxy:latest + networks: + - default + deploy: + mode: replicated + replicas: 2 + labels: + - "traefik.enable=true" + - "traefik.http.routers.proxy.entrypoints=web" + - "traefik.http.routers.proxy.rule=Host(`campus-proxy.my.domain`)" # todo pick a url + - "traefik.http.routers.proxy.middlewares=webs-redirectscheme" + - "traefik.http.routers.proxy-service=proxy-secure" + + - "traefik.http.routers.proxy-secure.entrypoints=webs" + - "traefik.http.routers.proxy-secure.tls=true" + - "traefik.http.routers.proxy-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.proxy-secure.rule=Host(`campus-proxy.my.domain`)" # todo pick a url + - "traefik.http.routers.proxy-secure.service=proxy-secure" + + - "traefik.http.services.proxy-secure.loadbalancer.server.port=8020" + googleSiteVerification: + image: nginx:latest + volumes: + # todo: use your own google verification file + - /srv/cephfs/livestream/TUM-Live/deployment/google695ffe94aec91c5d.html:/usr/share/nginx/html/google695ffe94aec91c5d.html + deploy: + mode: replicated + replicas: 2 + labels: + - "traefik.enable=true" + + - "traefik.http.routers.gsv-secure.entrypoints=webs" + - "traefik.http.routers.gsv-secure.tls=true" + - "traefik.http.routers.gsv-secure.tls.certresolver=liveresolver" + - "traefik.http.routers.gsv-secure.rule=Host(`live.rbg.tum.de`) && Path(`/google695ffe94aec91c5d.html`)" + - "traefik.http.routers.gsv-secure.service=gsv-secure" + + - "traefik.http.services.gsv-secure.loadbalancer.server.port=80" + +volumes: + recordings: + persist: + mariadb_data: + workerlog: + meilisearch: + grafana: + prometheus: + +networks: + agent_network: + driver: overlay + attachable: true + default: + driver: overlay + host: + name: host + external: true diff --git a/docs/static/deployment/traefik.toml b/docs/static/deployment/traefik.toml new file mode 100644 index 000000000..24e38618e --- /dev/null +++ b/docs/static/deployment/traefik.toml @@ -0,0 +1,38 @@ +[entryPoints] + [entryPoints.web] + address = ":80" + [entryPoints.webs] + address = ":443" + [entryPoints.rtmp] + address = ":1935" + +[api] + dashboard = true + insecure = true + +[providers.docker] + endpoint = "unix:///var/run/docker.sock" + exposedByDefault = false + swarmMode = true + +[accessLog] + filePath = "/var/log/traefik/access.log" + bufferingSize = 20 + [accessLog.fields.headers] + defaultMode = "keep" + [accessLog.fields.headers.names] + "User-Agent" = "keep" + "Authorization" = "drop" + "Content-Type" = "drop" + +[log] + level = "INFO" + +[certificatesResolvers.liveresolver.acme] + email = "some.name@mail.de" + storage = "/acme/acme-v2.json" + [certificatesResolvers.liveresolver.acme.httpChallenge] + entryPoint = "web" + +[metrics] + [metrics.prometheus] \ No newline at end of file