Initial commit

CREDITS

@@ -0,0 +1,37 @@
+cue3bot
+  3720	Matt Chambers
+   167	John Welborn
+   123	Michael Zhang
+    50	J Robert Ray
+    27	Bond-Jay Ting
+    22	Kasra Faghihi
+    14	Blair Zajac
+     2	Kevin Coats
+     1	Jordon Phillips
+
+rqd
+   473	John Welborn
+    25	J Robert Ray
+     6	Yudi Xue
+     2	Blair Zajac
+     2	Michael Zhang
+     2	Jordon Phillips
+     1	Kasra Faghihi
+
+spi_cue
+   252	John Welborn
+   226	Matt Chambers
+    12	Jordon Phillips
+     8	Yudi Xue
+     5	J Robert Ray
+     4	Michael Zhang
+     3	Blair Zajac
+
+python_ice_server
+    27	Blair Zajac
+     8	Cottalango Leon
+     5	J Robert Ray
+     3	John Welborn
+     2	Michael Zhang
+     1	Geo Snelling
+     1	Sam Richards

LICENSE

@@ -0,0 +1,13 @@
+Copyright 2018 Sony Pictures Imageworks Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

README

@@ -0,0 +1,55 @@
+
+Introduction:
+~~~~~~~~~~~~~
+
+Cue3 is the render farm management software used at Imageworks. It allows users
+to submit jobs to a configurable dispatch queue that allocates the necessary
+computational resources. Cue3 has been in use at Imageworks since 2007.
+
+Basic Queue Vocabulary:
+~~~~~~~~~~~~~~~~~~~~~~~
+
+Queue - The renderfarm as used to actively process a large number of render
+jobs using define priorities.
+
+Cue - A nickname for the entire queuing system. The base name of the
+SPI render queue tools. Thus "cuetopia3", "cuerun", "cueman3", etc.
+
+Cores - The processing units that make up a render machine. A machine
+known as a "dub" contains 8 cpu cores.
+
+Procs - Old terminology from cue2 where it use to refer to a single
+core of a machine. Typically a proc refers to a running frame.
+
+Job - Any script sent to the queue to be processed on remote cores.
+
+Layers - The sub-jobs in an outline script job.
+
+Frames - An individual command contained in a layer.
+
+Dependent Job - a job which will not run until the frames of
+another job are completed.
+
+Soft Dependency - when all the frames of the first job need to
+finish for the second job to begin.
+
+Hard Dependency - a frame to frame dependency (only the corresponding
+frames need to finish).
+
+cuetopia3 - A graphical user interface to monitor and interact with
+Jobs, Layers and Frames.
+
+Outline Script - A powerful tool for submitting jobs to the queue. Outline
+scripts can batch what would be multiple job submissions into a single job,
+setting up dependencies and/or running in parallel. Outline scripts can
+submit almost any type of job to the cue - maya, katana, even shell commands.
+
+pycuerun - The command used to submit outline scripts to the queue.
+booked - When machine resources are assigned to a job causing a
+frame to start running.
+
+Note:
+~~~~~
+
+This project is missing some components that we couldn't open source and is not
+yet expected to build. But stay tuned for more information and development.

cue3bot/.gitignore

@@ -0,0 +1,12 @@
+/build
+/classes-slice
+/classes-test
+/dist
+/logs
+/src-slice
+/target
+/velocity.log
+
+/.idea
+/nbproject
+/NETBEANS_CLASSPATH_SPECIFIC

cue3bot/.gitlab-ci.yml

@@ -0,0 +1,22 @@
+variables:
+  GIT_SUBMODULE_STRATEGY: normal
+
+test:
+  stage: test
+  script: bin/sbt -Dsbt.log.noformat=true clean test
+  artifacts:
+    paths:
+    - target/
+    expire_in: 1 hour
+
+pack:
+  stage: pack
+  script: bin/sbt -Dsbt.log.noformat=true pack
+  artifacts:
+    paths:
+    - target/pack
+    expire_in: 1 week
+
+stages:
+  - test
+  - pack

cue3bot/bin/check_ice.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+
+
+#  Copyright (c) 2018 Sony Pictures Imageworks Inc.
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+
+import sys
+import socket
+import os
+
+try:
+    host = sys.argv[1]
+    object = sys.argv[2]
+    port = sys.argv[3]
+except:
+    print "usage: check_ice.py host object port"
+    sys.exit(1)
+
+import Ice
+
+try:
+    ic = Ice.initialize(["--Ice.ImplicitContext=Shared", "--Ice.MessageSizeMax=10240"])
+    implicit_context = ic.getImplicitContext()
+    implicit_context.put('argv', 'xyzzy')
+    implicit_context.put('hostname', str(socket.gethostname()))
+    implicit_context.put('pid', str(os.getpid()))
+    implicit_context.put('username', 'nagios')
+    proxy = ic.stringToProxy("%s:default -h %s -p %s" % (object, host, port))
+    proxy.ice_ping()
+except Exception, e:
+    print e
+    sys.exit(1)
+finally:
+    ic.destroy()
+
+
+print "%s ice object is available" % object

cue3bot/bin/sbt

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+# ORACLE_HOME should match project/build.scala
+export ORACLE_HOME=/usr/lib64/oracle-instantclient-11.2
+export LD_LIBRARY_PATH=/shots/spi/home/java/FileSequence/x86_64/v6/rhel7/:${ORACLE_HOME}
+export TNS_ADMIN=/opt/tns
+
+project_dir=$(readlink -f $(dirname $0)/..)
+
+exec \
+  /usr/bin/java \
+    -Dcatalina.base=$project_dir \
+    -XX:+UseParNewGC \
+    -XX:+UseConcMarkSweepGC \
+    -XX:+CMSClassUnloadingEnabled \
+    -XX:CMSInitiatingOccupancyFraction=70 \
+    -verbose:gc \
+    -XX:+PrintGCDateStamps \
+    -XX:+PrintGCDetails \
+    -XX:+PrintGCTimeStamps \
+    -XX:+PrintHeapAtGC \
+    -XX:+PrintTenuringDistribution \
+    -Xloggc:${project_dir}/logs/gc.log \
+    -XX:+UseGCLogFileRotation \
+    -XX:NumberOfGCLogFiles=10 \
+    -XX:GCLogFileSize=1M \
+    -Dfile.encoding=UTF8 \
+    -Djava.net.preferIPv4Stack=true \
+    -Xms3G \
+    -Xmx3G \
+    -XX:+CMSClassUnloadingEnabled \
+    -XX:ReservedCodeCacheSize=192m \
+    $EXTRA_JVM_OPTS \
+    -jar `dirname $0`/sbt-launch.jar \
+      "$@"

cue3bot/build.sbt

@@ -0,0 +1,131 @@
+import sbtzerocice.{ZeroCIcePlugin=>Ice}
+import scalariform.formatter.preferences._
+
+// Disable scala!
+crossPaths := false
+autoScalaLibrary := false
+
+packSettings
+
+patchPack
+
+packMain := Map("cuebot" -> "com.imageworks.spcue.CuebotApplication")
+
+packGenerateWindowsBatFile := false
+
+packExcludeJars := Seq("scala-compiler-.*\\.jar")
+
+packJvmOpts := Map("cuebot" -> Seq(
+    "-server",
+    "-Xms3g",
+    "-Xmx3g",
+    "-XX:MaxDirectMemorySize=9223372036854775807",
+    "-Dnet.spy.log.LoggerImpl=net.spy.log.Log4JLogger",
+    "-Dhttp.proxyHost=proxy.spimageworks.com",
+    "-Dhttp.proxyPort=8080",
+    "-Dhttps.proxyHost=proxy.spimageworks.com",
+    "-Dhttps.proxyPort=8080",
+    "-Djava.net.preferIPv4Stack=true",
+    // Garbage collection logging.
+    "-verbose:gc",
+    "-XX:+PrintGCDateStamps",
+    "-XX:+PrintGCDetails",
+    "-XX:+PrintGCTimeStamps",
+    "-XX:+PrintHeapAtGC",
+    "-XX:+PrintTenuringDistribution",
+    "-Xloggc:${PROG_HOME}/logs/gc.log",
+    "-XX:+UseGCLogFileRotation",
+    "-XX:NumberOfGCLogFiles=10",
+    "-XX:GCLogFileSize=1M",
+    // Garbage collection configuration.
+    "-XX:+UseParNewGC",
+    "-XX:+UseConcMarkSweepGC",
+    "-XX:+CMSClassUnloadingEnabled",
+    "-XX:CMSInitiatingOccupancyFraction=70",
+    "-Djava.library.path=${ORACLE_HOME}:/shots/spi/home/java/FileSequence/x86_64/v6/rhel7"
+))
+
+net.virtualvoid.sbt.graph.Plugin.graphSettings
+
+genSliceSettings
+
+scalariformSettings
+
+ScalariformKeys.preferences := FormattingPreferences().
+  setPreference(AlignParameters, true).
+  setPreference(CompactControlReadability, true).
+  setPreference(DoubleIndentClassDeclaration, true)
+
+resolvers ++= Seq(
+    "spymemcached" at "http://files.couchbase.com/maven2/",
+    "zeroc" at "http://www.zeroc.com:8081/nexus/service/local/repositories/releases/content/",
+    "SPI" at "http://www.spimageworks.com/maven/"
+)
+
+{
+  val activemq_ver = "5.12.0"
+  // This should match the version spring_boot is trying to pull in.
+  val spring_ver = "4.2.8.RELEASE"
+  val spring_boot_ver = "1.3.8.RELEASE"
+  libraryDependencies ++= Seq(
+    "com.google.code.gson" % "gson" % "2.3.1",
+    "com.google.guava" % "guava" % "12.0.1",
+    "com.sun.mail" % "mailapi" % "1.5.4",
+    "commons-lang" % "commons-lang" % "2.6",
+    "com.zeroc" % "ice" % "3.6.1",
+    "org.apache.activemq" % "activemq-pool" % activemq_ver,
+    "org.apache.velocity" % "velocity" % "1.7",
+    "org.jdom" % "jdom" % "1.1.3",
+    "org.springframework.boot" % "spring-boot-starter-jdbc" % spring_boot_ver exclude ("org.springframework.boot", "spring-boot-starter-logging"),
+    "org.springframework.boot" % "spring-boot-starter-log4j" % spring_boot_ver,
+    "org.springframework.boot" % "spring-boot-starter-tomcat" % spring_boot_ver % "provided",
+    "org.springframework.boot" % "spring-boot-starter-web" % spring_boot_ver,
+    "org.springframework" % "spring-context-support" % spring_ver,
+    "org.springframework" % "spring-jms" % spring_ver,
+    "org.quartz-scheduler" % "quartz" % "2.2.1" exclude ("c3p0", "c3p0"),
+//    "javax.inject" % "javax.inject" % "1",
+//    "javax.servlet" % "javax.servlet-api" % "3.0.1" % "provided",
+//    "javax.mail" % "javax.mail-api" % "1.5.4",
+    "junit" % "junit" % "4.12" % "test",
+    // https://github.com/sbt/junit-interface
+    "com.novocode" % "junit-interface" % "0.11" % "test",
+    "org.springframework.boot" % "spring-boot-starter-test" % spring_boot_ver % "test"
+  )
+}
+
+// reduce the maximum number of errors shown by the Scala compiler
+maxErrors := 5
+
+// otherwise we get port conflicts in the tests
+parallelExecution in Test := false
+
+fork in Test := true
+
+// increase warnings generated by the Scala compiler
+scalacOptions ++= Seq(
+    "-Xlint",
+    "-Ywarn-dead-code",
+    "-deprecation",
+    "-feature",
+    "-unchecked")
+
+javacOptions ++= Seq(
+    "-Xlint:deprecation",
+    "-Xlint:unchecked")
+
+// ask ScalaTest for full stack traces
+// testOptions in Test += Tests.Argument("-oF")
+
+// name the war file 'cuebot.war'
+artifactName := { (config: sbt.ScalaVersion, module: ModuleID, artifact: Artifact) =>
+    artifact.name + "." + artifact.extension
+}
+
+// Faster incremental compilation
+incOptions := incOptions.value.withNameHashing(true)
+
+deployTask := {
+    "fab hot_redeploy:build=no" !
+}
+
+deployTask <<= deployTask.dependsOn(xerial.sbt.Pack.packArchiveTbz)