Test out scraping from Catagolue to dump into GliderDB

src/commonMain/kotlin/patterns/Spaceship.kt

@@ -14,20 +14,21 @@ import kotlin.math.abs
 fun fromGliderDBEntry(entry: String): Spaceship {
     val tokens = entry.split(":")
 
-    val period = tokens[4].toInt()
+    val period = tokens[4].split("/").first().toInt()
     val dx = tokens[5].toInt()
     val dy = tokens[6].toInt()
 
     val grid = SparseGrid(tokens.last(), rule=fromRulestring(tokens[2]))
     val phases = Array(period+1) { grid.step(1).deepCopy() }
 
-    val spaceship = Spaceship(period, dx, dy, phases)
+    val spaceship = Spaceship(dx, dy, period, phases)
+
     // TODO fix this
-    // require((spaceship.ruleRange!!.first as RuleFamily).rulestring == tokens[2]) { 
-    //     "Incorrect minimum rule! Got ${tokens[2]} instead of ${(spaceship.ruleRange!!.first as RuleFamily).rulestring} for ${tokens.last()}" 
+    // require((spaceship.ruleRange!!.first as RuleFamily).rulestring == tokens[2]) {
+    //     "Incorrect minimum rule! Got ${tokens[2]} instead of ${(spaceship.ruleRange!!.first as RuleFamily).rulestring} for ${tokens.last()}"
     // }
-    // require((spaceship.ruleRange!!.second as RuleFamily).rulestring == tokens[3]) { 
-    //     "Incorrect maximum rule! Got ${tokens[3]} instead of ${(spaceship.ruleRange!!.second as RuleFamily).rulestring} for ${tokens.last()}" 
+    // require((spaceship.ruleRange!!.second as RuleFamily).rulestring == tokens[3]) {
+    //     "Incorrect maximum rule! Got ${tokens[3]} instead of ${(spaceship.ruleRange!!.second as RuleFamily).rulestring} for ${tokens.last()}"
     // }
 
     spaceship.name = tokens[0]
@@ -62,6 +63,26 @@ open class Spaceship(val dx: Int, val dy: Int, val period: Int, val phases: Arra
         }
     }
 
+    /**
+     * The spaceship's simplified speed outputted as a tuple of (dx, dy, p)
+     */
+    val simplifiedSpeed by lazy {
+        fun gcd(a: Int, b: Int): Int {
+            var num1 = a
+            var num2 = b
+            while (num2 != 0) {
+                val temp = num2
+                num2 = num1 % num2
+                num1 = temp
+            }
+
+            return num1
+        }
+
+        val factor = gcd(gcd(abs(dx), abs(dy)), period)
+        return@lazy Pair(Pair(minOf(abs(dx), abs(dy)) / factor, maxOf(abs(dx), abs(dy)) / factor), period / factor)
+    }
+
     /**
      * The spaceship's direction (orthogonal, diagonal, knight, etc.)
      */
@@ -189,7 +210,7 @@ open class Spaceship(val dx: Int, val dy: Int, val period: Int, val phases: Arra
 
     /**
      * Checks if 2 spaceships are the same
-     * @param The other spaceship
+     * @param other The other spaceship
      * @return Returns true if the spaceships are the same, false otherwise
      */
     override fun equals(other: Any?): Boolean {

src/commonMain/kotlin/patterns/gliderdb/GliderDB.kt

@@ -16,10 +16,13 @@ import rules.RuleRangeable
  * @param lst A list of spaceships to initialise the database with.
  */
 class GliderDB<R>(lst: List<Spaceship>): PatternCollection<Spaceship>() where R : RuleFamily, R : RuleRangeable<R> {
+    /**
+     * The list containing all the spaceships.
+     */
     val lst = ArrayList(lst)
 
     /**
-     * Builds the GliderDB database from a string
+     * Builds the GliderDB database from a string.
      */
     constructor(string: String): this(string.split("! ").map { fromGliderDBEntry(it) })
 
@@ -57,7 +60,7 @@ class GliderDB<R>(lst: List<Spaceship>): PatternCollection<Spaceship>() where R
 
     /**
      * Searches for spaceships moving at ([dx], [dy])c/[period].
-     * @param higher_periods Should ships of the same speed but higher period be returned?
+     * @param higherPeriod Should ships of the same speed but higher period be returned?
      */
     fun searchBySpeed(dx: Int, dy: Int, period: Int, higherPeriod: Boolean = false): GliderDB<R> {
         if (higherPeriod) {
@@ -77,12 +80,19 @@ class GliderDB<R>(lst: List<Spaceship>): PatternCollection<Spaceship>() where R
         }
     }
 
+    /**
+     * Searches for spaceships moving at [speed].
+     * @param higherPeriod Should ships of the same speed but higher period be returned?
+     */
     fun searchBySpeed(speed: String, higherPeriod: Boolean = false): GliderDB<R> {
         val (displacement, period) = parseSpeed(speed)
         val (dx, dy) = displacement
         return searchBySpeed(dx, dy, period, higherPeriod)
     }
 
+    /**
+     * Searches for spaceships moving along the slope ([dx], [dy]).
+     */
     fun searchBySlope(dx: Int, dy: Int): GliderDB<R> {
         require(dx != 0 || dy != 0) { "(0, 0) is not a valid slope." }
         return GliderDB(
@@ -94,19 +104,54 @@ class GliderDB<R>(lst: List<Spaceship>): PatternCollection<Spaceship>() where R
         )
     }
 
-    fun searchByRule(rule: RuleFamily) = GliderDB<R>(
-        this.filter { rule in it.ruleRange!! }
+    /**
+     * Searches for ships that work in [rule].
+     */
+    fun searchByRule(rule: R) = GliderDB<R>(
+        this.filter { rule.between(it.ruleRange!!.minRule as R, it.ruleRange!!.maxRule as R) }
     )
 
+    /**
+     * Searches for ships that work in [ruleRange].
+     */
     fun searchByRule(ruleRange: RuleRange<R>) = GliderDB<R>(
         this.filter { 
             ruleRange intersect (it.ruleRange!! as RuleRange<R>) != null
         }
     )
 
-    // Redundancy check
-    fun checkRedundant(): List<Spaceship> {
-        TODO("Not yet implemented")
+    /**
+     * Checks if a given [spaceship] is redundant or makes another redundant and
+     * outputs the other spaceship.
+     */
+    fun checkRedundant(spaceship: Spaceship): List<Pair<Spaceship, Spaceship>> {
+        val output = ArrayList<Pair<Spaceship, Spaceship>>()
+        for (i in lst.indices) {
+            if (lst[i].simplifiedSpeed == spaceship.simplifiedSpeed) {
+                // Check if the rule ranges are the same
+                if (lst[i].ruleRange!! == spaceship.ruleRange!!) {
+                    if (lst[i].canonPhase.bounds.area > spaceship.canonPhase.bounds.area)
+                        output.add(Pair(lst[i], spaceship))
+                    else
+                        output.add(Pair(spaceship, lst[i]))
+                } else {
+                    // Check if either rule range contains the other
+                    val intersection =
+                        lst[i].ruleRange!! as RuleRange<R> intersect spaceship.ruleRange!! as RuleRange<R>
+                    if (intersection == lst[i].ruleRange) {
+                        // The ship is only redundant if it covers both a smaller rule range and has a smaller bounding box
+                        if (lst[i].canonPhase.bounds.area < spaceship.canonPhase.bounds.area)
+                            output.add(Pair(lst[i], spaceship))
+                    } else if (intersection == spaceship.ruleRange) {
+                        // The ship is only redundant if it covers both a smaller rule range and has a smaller bounding box
+                        if (lst[i].canonPhase.bounds.area > spaceship.canonPhase.bounds.area)
+                            output.add(Pair(lst[i], spaceship))
+                    }
+                }
+            }
+        }
+
+        return output
     }
 
     // Reading and writing database to and from a string

src/commonMain/kotlin/rules/Rule.kt

@@ -101,7 +101,7 @@ abstract class Rule {
      * The lookup table for 2x2 tiles
      */
     @OptIn(ExperimentalUnsignedTypes::class)
-    protected val lookup2x2 = ULongArray(2.0.pow(17).toInt()) { ULong.MAX_VALUE }
+    protected val lookup2x2 = ULongArray(2.0.pow(1).toInt()) { ULong.MAX_VALUE }
 
     /**
      * The possible successor cell states of each cell state

src/commonMain/kotlin/rules/RuleRange.kt

@@ -36,4 +36,15 @@ class RuleRange<R>(val minRule: R, val maxRule: R) : Sequence<RuleFamily>
     infix fun intersect(ruleRange: RuleRange<R>): RuleRange<R>? = minRule.intersect(this, ruleRange)
 
     override fun iterator(): Iterator<RuleFamily> = enumerationIterator
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other !is RuleRange<*>) return false
+
+        return minRule == other.minRule && maxRule == other.maxRule
+    }
+
+    override fun toString(): String {
+        return "$minRule - $maxRule"
+    }
 }

src/commonMain/kotlin/rules/RuleRangeable.kt

@@ -1,5 +1,8 @@
 package rules
 
+/**
+ * Represents rule families on which rule ranges can be defined. Should only be implemented by a RuleFamily [R].
+ */
 interface RuleRangeable<R> where R : RuleFamily, R : RuleRangeable<R> {
     /**
      * Checks if a rule is between 2 other rules

src/commonMain/kotlin/rules/hrot/HROTGenerations.kt

@@ -406,4 +406,20 @@ class HROTGenerations : BaseHROT, RuleRangeable<HROTGenerations> {
 
     private fun newRuleWithTransitions(birth: Iterable<Int>, survival: Iterable<Int>): HROTGenerations =
         HROTGenerations(birth, survival, numStates, neighbourhood[0], weights, stateWeights)
+
+    override fun equals(other: Any?): Boolean {
+        if (this === other) return true
+        if (other == null || this::class != other::class) return false
+
+        other as HROTGenerations
+
+        if (birth != other.birth) return false
+        if (survival != other.survival) return false
+        if (numStates != other.numStates) return false
+        if (!neighbourhood.contentEquals(other.neighbourhood)) return false
+        if (!weights.contentEquals(other.weights)) return false
+        if (!stateWeights.contentEquals(other.stateWeights)) return false
+
+        return true
+    }
 }

src/jvmMain/kotlin/Main.jvm.kt

@@ -1,31 +1,84 @@
-import it.skrape.core.*
-import it.skrape.fetcher.*
-import it.skrape.selects.*
-import it.skrape.selects.html5.*
-import kotlinx.io.buffered
-import kotlinx.io.files.Path
-import kotlinx.io.files.SystemFileSystem
-import kotlinx.io.writeString
+import it.skrape.core.htmlDocument
+import it.skrape.fetcher.HttpFetcher
+import it.skrape.fetcher.response
+import it.skrape.fetcher.skrape
+import it.skrape.selects.eachHref
+import it.skrape.selects.html5.a
+import it.skrape.selects.html5.body
 import patterns.Spaceship
+import patterns.gliderdb.GliderDB
 import rules.hrot.HROT
 import rules.hrot.HROTGenerations
-import rules.nontotalistic.rules.DeficientINT
-import rules.nontotalistic.rules.INT
-import rules.nontotalistic.rules.INTGenerations
-import rules.nontotalistic.transitions.R2VonNeumannINT
-import rules.ruleloader.builders.ruletable
-import rules.ruleloader.ruletableFromFile
-import rules.ruleloader.ruletree.ruletreeDirectiveFromString
-import search.cfind.CFind
-import search.cfind.SearchStrategy
-import search.cfind.ShipSymmetry
-import simulation.*
-import java.io.File
-import kotlin.random.Random
-import patterns.Oscillator
-import rules.hrot.HROTExtendedGenerations
+import simulation.DenseGrid
+import simulation.SparseGrid
 
 actual fun main() {
+    val gliderdb = GliderDB<HROTGenerations>(
+        skrape(HttpFetcher) {
+            request {
+                url = "https://raw.githubusercontent.com/jedlimlx/gliderdb-reader/main/public/R1-C3-NM-gliders.db.txt"
+            }
+            response {
+                htmlDocument { body { findFirst { text } } }
+            }
+        }
+    )
+//    println(gliderdb.searchByRule(HROTGenerations("/2/3")).map {
+//        "x = 0, y = 0, rule = ${it.ruleRange!!.minRule}\n${it.canonPhase}"
+//    }.joinToString("\n\n"))
+
+    val rules = skrape(HttpFetcher) {
+        request { url = "https://catagolue.hatsya.com/rules/generations" }
+        response { htmlDocument { a { findAll { eachHref } } } }
+    }.filter { Regex("/census/g3b[0-8]+s[0-8]+").matches(it) }.map { it.split("/").last() }
+    for (rulestring in rules) {
+        val rule = HROTGenerations(rulestring)
+        val symmetries = skrape(HttpFetcher) {
+            request { url = "https://catagolue.hatsya.com/census/$rulestring" }
+            response { htmlDocument { a { findAll { eachHref } } } }
+        }.filter { "/census/$rulestring" in it }.map { it.split("/").last() }
+
+        for (symmetry in symmetries) {
+            val list: List<String> = skrape(HttpFetcher) {
+                request { url = "https://catagolue.hatsya.com/census/$rulestring/$symmetry" }
+                response { htmlDocument { a { findAll { eachHref } } } }
+            }.filter { Regex("/census/$rulestring/$symmetry/xq[0-9]+").containsMatchIn(it) }
+
+            val ships = list.map {
+                "https://catagolue.hatsya.com/textcensus/$rulestring/$symmetry/" + it.split("/").last()
+            }.map {
+                skrape(HttpFetcher) {
+                    request { url = it }
+                    response { htmlDocument { body { findFirst { text } } } }
+                }.split(" ")
+            }.map {
+                it.subList(1, it.size).map {
+                    it.split(",").first().replace("\"", "")
+                }.filter { it[0] == 'x' }
+            }.map {
+                it.map { DenseGrid(rule=rule, pattern=it).identify() as Spaceship }
+            }.flatten()
+
+            println("Checking $rulestring...")
+            println("-".repeat(30))
+
+            val smallerDB = gliderdb.searchByRule(rule)
+            smallerDB.forEach { println("$it, ${it.ruleRange}") }
+            println()
+
+            for (ship in ships) {
+                val output = smallerDB.checkRedundant(ship)
+                if (output.isEmpty()) {
+                    println("Added $ship, ${ship.ruleRange}")
+                    gliderdb.add(ship)
+                    smallerDB.add(ship)
+                }
+            }
+
+            println()
+        }
+    }
+
     // val rule = HROT("R2,C2,S6-9,14-20,B7-8,15-24,NM")
     // val rulestring = "r2bffc0c0s0fe1e0"
     // val symmetry = "cfind_stdin"
@@ -60,23 +113,6 @@ actual fun main() {
     //     }
     // }.flatten().filter { it.dx != 0 || it.dy != 0 }
 
-    // val gliderdb = GliderDB(output)
-    // println(gliderdb.searchByRule(HROT("R2,C2,S6-9,B7-8,NM")..HROT("R2,C2,S6-9,14-24,B7-8,NM")))
-
-    // val output = skrape(HttpFetcher) {
-    //     request {
-    //         url = "https://raw.githubusercontent.com/jedlimlx/gliderdb-reader/main/src/assets/R1-C3-NM-gliders.db.txt"
-    //     }
-    //     response {
-    //         htmlDocument { body { findFirst { text } } }
-    //     }
-    // }
-
-    // val gliderdb = GliderDB(output)
-    // println(gliderdb.searchByRule(HROTGenerations("/2/3")..HROTGenerations("012345678/2345678/3")).map {
-    //     "x = 0, y = 0, rule = ${it.ruleRange!!.first}\n${it.canonPhase}"
-    // }.joinToString("\n\n"))
-
     // val transitions: MutableList<List<Int>> = arrayListOf()
     // val weights = arrayOf(3, 3, 3, 3, 3, 3, 3, 3, 2, 2, 2, 2)
     // for (i in 0..<(1 shl 12)) {
@@ -90,24 +126,6 @@ actual fun main() {
 
     // println(R2VonNeumannINT(transitions).transitionString)
 
-    // val pattern = "b3o\$2o2bo\$2o2bo\$b4o\$2b2o9\$2b2o\$b4o\$2o2bo\$2o2bo\$b3o!"
-    // val ship = SparseGrid("b3o\$2o2bo\$2o2bo\$b4o\$2b2o!", HROT("R2,C2,S6-9,B7-8,NM")).identify()!!
-
-    // var count = 0
-    // val range = ship.ruleRange!!.first as HROT .. ship.ruleRange!!.second as HROT
-    // //println("${range.size} rules to search.")
-    // for (i in range.randomSequence()) {
-    //     val test = DenseGrid(pattern, i)
-    //     val output = test.identify(200)
-    //     if (output != null && (output as Spaceship).period > 1)
-    //         println("$i, ${output}")
-
-    //     if ((count++).mod(1000) == 0)
-    //         println("Searched $count...")
-    // }
-
-    val ruletable = ruletableFromFile("SoManyShips3.rule")
-
     // B2-ei3cjkr4cektyz5-cnr6-ik78/S01e2-ae3cnqry4cqrtwyz5-ain6ekn7e
     // B2ac3anr4-ijkz5cjkry6-cn7c8/S12i3aejy4nqtw5ceny6-kn7c
     // HROT("R2,C2,S6-11,B4,9-11,NW0020003330230320333000200")
@@ -117,15 +135,4 @@ actual fun main() {
 //        //direction = Coordinate(1, 1), lookaheadDepth = 3, numShips = 30
 //    )
 //    search.search()
-
-//    val search = CFind(
-//        HROT("R2,C2,S6-9,B7-8,NM"),
-//        2, 1, 8, symmetry = ShipSymmetry.ASYMMETRIC,
-//        verbosity = 1, searchStrategy = SearchStrategy.HYBRID_BFS
-//    )
-//    search.search()
-
-    for (rule in HROT("B3/S23")..HROT("B345/S23")) {
-        println(rule)
-    }
 }