Merge pull request #811 from munen/integration

Next Release

app/assets/stylesheets/_start.scss

@@ -148,7 +148,7 @@ button {
         text-align: center;
     }
     background: $talk-dark;
-    
+
 }
 
 .pages-publish_talk {
@@ -374,9 +374,9 @@ h1, h2 {
 #publishers {
     #publisher-collection {
         .publisher {
-            padding: 24px 12px;
+            /*padding: 24px 12px;*/
             float:left;
-            margin-bottom: 3px;
+            /*margin-bottom: 3px;*/
             h4 {
                 font-family: $body-font-family;
                 font-weight: 700;
@@ -527,5 +527,5 @@ h1, h2 {
                 // button {
                 //     height: 60px;
                 // }
-           
-            }
+
+            }

app/models/talk.rb

@@ -504,7 +504,7 @@ def archive_from_dump!
 
   # venue.stored_file('dump_1481635942').content_length
   def debug_processing
-    puts "Startet at:         #{sarted_at}"
+    puts "Startet at:         #{started_at}"
     puts "Ended at:           #{ended_at}"
     puts "Effective duration: #{effective_duration}"
     puts

app/models/venue.rb

@@ -194,6 +194,10 @@ def icecast_callback_url
     Settings.icecast.callback_url
   end
 
+  def transcoding_script_url
+    Settings.icecast.transcoding_script_url
+  end
+
   def icecast_params
     {
       public_ip_address: public_ip_address,
@@ -467,7 +471,8 @@ def unprovision_test
   end
 
   def provision
-    assign_attributes( source_password: generate_password,
+    assign_attributes( source_password:
+                         Settings.icecast.stoical_password || generate_password,
                        admin_password: generate_password,
                        client_token: generate_client_token,
                        mount_point: generate_mount_point )

bin/backup_database

@@ -3,7 +3,11 @@
 require 'yaml'
 require 'fileutils'
 
-config_path = File.expand_path(File.join(%w(.. .. config settings.local.yml)), __FILE__)
+rel_path = File.join(%w(.. .. config settings.local.yml))
+
+config_path = File.expand_path(rel_path, __FILE__)
+
+puts "Reading config from #{config_path}"
 
 config = YAML.load(File.read(config_path))
 
@@ -12,13 +16,21 @@ secret_access_key = config['fog']['storage']['aws_secret_access_key']
 
 bucket, region = config['storage']['database_backups'].split('@')
 
+puts "Using bucket #{bucket} in region #{region}"
+
 backup_name = "dumpall-#{Date.today.strftime('%Y%m%d')}.sql.gz"
 
 backup_path = File.expand_path(File.join(%w(.. .. tmp db)), __FILE__)
 
+puts "Creating backup #{backup_name} in #{backup_path}"
+
 FileUtils.mkdir_p backup_path
 
 Dir.chdir backup_path do
+  puts "Collecting data..."
   system("sudo -u postgres pg_dumpall | gzip > #{backup_name}")
-  system("AWS_ACCESS_KEY_ID=#{access_key_id};AWS_SECRET_ACCESS_KEY=#{secret_access_key};aws s3 mv #{backup_name} s3://#{bucket}/ --region #{region}")
+  puts "Uploading data..."
+  system("AWS_ACCESS_KEY_ID=#{access_key_id} AWS_SECRET_ACCESS_KEY=#{secret_access_key} aws s3 mv #{backup_name} s3://#{bucket}/ --region #{region}")
 end
+
+puts "Done."

bin/stream.liq

@@ -0,0 +1,23 @@
+#!/usr/bin/liquidsoap -v
+
+set("log.file.path", "./liquidsoap.log")
+set("log.stdout", true)
+
+icecast_host = argv(1)
+auido_file = argv(2)
+
+icecast_port = 80
+icecast_mount = "/live"
+icecast_password = "12345"
+
+source = single(audio_file)
+
+ignore(output.icecast(
+    %vorbis(channels=1),
+    host=icecast_host,
+    port=icecast_port,
+    mount=icecast_mount,
+    password=icecast_password,
+    public=false,
+    mean(source)
+))

config/schedule.rb

@@ -37,9 +37,9 @@
   rake "validity:check"
 end
 
-#every 3.hours, roles: [:app] do
-#  rake 'talks:popularity'
-#end
+every 60.minutes, roles: [:app] do
+  rake 'talks:popularity'
+end
 
 every 24.hours, at: '2:00 am', roles: [:app] do
   runner "Metric.snapshot!"

config/settings.yml

@@ -168,7 +168,9 @@ icecast:
     #image: ami-fbc21694 (icebox 3.19 consistent uid/gid)
     #image: ami-ecd60583 (icebox 3.20 liquidsoap)
     #image: ami-bf4d9ed0 (icebox 3.21 improved live transcoding)
-    image: ami-bf4d9ed0
+    #image: ami-1ed57071 (icebox 3.22 fix transc. stereo signals + pull icebox.liq)
+    #image: ami-d6ca6fb9 (icebox 3.23 fixed docker issue)
+    image: ami-d6ca6fb9
     default_instance_type: t2.micro
     security_group: Icecast Servers
     key_name: phil-ffm

lib/icecast/Dockerfile

@@ -11,12 +11,14 @@ ENV DEBIAN_FRONTEND noninteractive
 RUN groupadd -r icecast && \
   useradd -r -g icecast icecast2
 
-RUN echo 'deb-src http://deb.debian.org/debian/ jessie-backports main' \
+RUN echo 'deb-src http://httpredir.debian.org/debian/ jessie-backports main' \
     >> /etc/apt/sources.list.d/backports.list
+
 RUN apt-get -qq -y update && \
   apt-get -qq -y install curl build-essential dpkg-dev libssl-dev && \
   apt-get -qq -y -t jessie-backports build-dep icecast2 && \
   apt-get -qq -y -t jessie-backports source icecast2
+
 RUN (cd icecast2-2.4.2; dpkg-buildpackage -b)
 RUN dpkg -i icecast2_2.4.2-1~bpo8+1_amd64.deb
 

lib/icecast/icebox.liq

@@ -56,7 +56,7 @@ stream0 = input.http(url,
                      # on_disconnect=input_disconnect
                      )
 
-stream1 = fallback(track_sensitive=false, [stream0, interlude1])
+stream1 = fallback(track_sensitive=false, [stream0, mean(stream0), interlude1])
 
 stream2 = strip_blank(stream1, max_blank=10.)
 #stream2 = strip_blank(stream1, max_blank=10., threshold=-50.)

lib/icecast/start.sh

@@ -18,6 +18,11 @@ ICECAST_ADMIN_USER="${ICECAST_ADMIN_USER:-admin}"
 ICECAST_PID=$!
 echo Icecast start with pid $ICECAST_PID
 
+# pull recent transcoding script from app server
+# this allows fine tuning the transcoding script
+# without having to roll out a new icebox
+curl $TRANSCODING_SCRIPT_URL > icebox.liq
+
 # start live transcoder
 ./run_liquidsoap.sh &
 

lib/tasks/fog.rake

@@ -140,19 +140,32 @@ def aws
 
   s3cmd_path = File.join(ENV['HOME'], '.s3cfg')
   awscli_path = File.join(ENV['HOME'], '.aws', 'credentials')
+  awscli_path2 = File.join(ENV['HOME'], '.aws', 'config')
 
   if File.exist?(s3cmd_path)
-    puts 'Found %s' % s3cmd_path
+    puts 'Reading credentials from %s' % s3cmd_path
     config = IniFile.load(s3cmd_path)
     access_key = config['default']['access_key']
     secret_key = config['default']['secret_key']
   elsif File.exist?(awscli_path)
-    puts 'Found %s' % awscli_path
+    puts 'Reading credentials from %s' % awscli_path
     config = IniFile.load(awscli_path)
     access_key = config['default']['aws_access_key_id']
     secret_key = config['default']['aws_secret_access_key']
+  elsif File.exist?(awscli_path2)
+    puts 'Reading credentials from %s' % awscli_path2
+    config = IniFile.load(awscli_path2)
+    access_key = config['default']['aws_access_key_id']
+    secret_key = config['default']['aws_secret_access_key']
   else
-    raise "could read neither #{s3cmd_path} nor #{awscli_path}"
+    puts "could read neither #{s3cmd_path} nor #{awscli_path}"
+    puts
+    puts "  sudo apt-get install python-pip libyaml-dev"
+    puts "  pip install --upgrade --user awscli"
+    puts "  aws configure"
+    puts "  aws configure set default.s3.signature_version s3v4"
+    puts
+    exit
   end
 
   @aws = Fog::Storage.new(provider: 'AWS',

lib/tasks/talks.rake

@@ -1,12 +1,24 @@
 namespace :talks do
 
-  desc "Save all archived talks to update popularity"
+  # NEWSCHOOL
   task popularity: :environment do
-    Talk.archived.find_each do |talk|
-      talk.save! # tiggers: `before_save :set_popularity`
-    end
+    sql = <<-EOF
+      UPDATE talks
+      SET popularity = (((play_count - 1) ^ 0.8)::real / (((floor((floor(extract(epoch from current_timestamp)) - floor(extract(epoch from processed_at)))/3600)) + 2) ^ 1.8)) * penalty
+      WHERE processed_at IS NOT NULL
+      AND play_count > 0;
+    EOF
+    ActiveRecord::Base.connection.execute(sql)
   end
 
+  # OLDSCHOOL
+  # desc "Save all archived talks to update popularity"
+  # task popularity: :environment do
+  #   Talk.archived.find_each do |talk|
+  #     talk.save! # tiggers: `before_save :set_popularity`
+  #   end
+  # end
+
   desc "Notify all participants of event about upcoming talk"
   task remind: :environment do
     Talk.prelive.find_each do |talk|

lib/templates/env.list.erb

@@ -3,6 +3,7 @@ ICECAST_ADMIN_PASSWORD=<%= admin_password %>
 ICECAST_LIMIT_SOURCES=12
 
 CALLBACK_URL=<%= icecast_callback_url %>
+TRANSCODING_SCRIPT_URL=<%= transcoding_script_url %>
 CLIENT_TOKEN=<%= client_token %>
 
 STATS_INTERVAL=4

lib/templates/userdata.sh.erb

@@ -28,7 +28,7 @@ if [ -f /sys/hypervisor/uuid ] && [ `head -c 3 /sys/hypervisor/uuid` == ec2 ]; t
 
   echo $AWS_CREDENTIALS > /etc/passwd-s3fs
   chmod 600 /etc/passwd-s3fs
-  /usr/local/bin/s3fs $BUCKET $STORAGE -o allow_other
+  /usr/local/bin/s3fs $BUCKET $STORAGE -o allow_other,nonempty
 
   # fix old permission issue
   # chown -R icecast2: $STORAGE/$PREFIX/