Initial Commit

src/main/kotlin/com/looker/droidify/index/IndexHandler.kt

@@ -0,0 +1,265 @@
+package com.looker.droidify.index
+
+import com.looker.droidify.entity.Product
+import com.looker.droidify.entity.Release
+import com.looker.droidify.utility.extension.android.*
+import org.xml.sax.Attributes
+import org.xml.sax.helpers.DefaultHandler
+import java.text.SimpleDateFormat
+import java.util.Locale
+import java.util.TimeZone
+
+class IndexHandler(private val repositoryId: Long, private val callback: Callback): DefaultHandler() {
+  companion object {
+    private val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US)
+      .apply { timeZone = TimeZone.getTimeZone("UTC") }
+
+    private fun String.parseDate(): Long {
+      return try {
+        dateFormat.parse(this)?.time ?: 0L
+      } catch (e: Exception) {
+        0L
+      }
+    }
+
+    internal fun validateIcon(icon: String): String {
+      return if (icon.endsWith(".xml")) "" else icon
+    }
+  }
+
+  interface Callback {
+    fun onRepository(mirrors: List<String>, name: String, description: String,
+      certificate: String, version: Int, timestamp: Long)
+    fun onProduct(product: Product)
+  }
+
+  internal object DonateComparator: Comparator<Product.Donate> {
+    private val classes = listOf(Product.Donate.Regular::class, Product.Donate.Bitcoin::class,
+      Product.Donate.Litecoin::class, Product.Donate.Flattr::class, Product.Donate.Liberapay::class,
+      Product.Donate.OpenCollective::class)
+
+    override fun compare(donate1: Product.Donate, donate2: Product.Donate): Int {
+      val index1 = classes.indexOf(donate1::class)
+      val index2 = classes.indexOf(donate2::class)
+      return when {
+        index1 >= 0 && index2 == -1 -> -1
+        index2 >= 0 && index1 == -1 -> 1
+        else -> index1.compareTo(index2)
+      }
+    }
+  }
+
+  private class RepositoryBuilder {
+    var address = ""
+    val mirrors = mutableListOf<String>()
+    var name = ""
+    var description = ""
+    var certificate = ""
+    var version = -1
+    var timestamp = 0L
+  }
+
+  private class ProductBuilder(val repositoryId: Long, val packageName: String) {
+    var name = ""
+    var summary = ""
+    var description = ""
+    var icon = ""
+    var authorName = ""
+    var authorEmail = ""
+    var source = ""
+    var changelog = ""
+    var web = ""
+    var tracker = ""
+    var added = 0L
+    var updated = 0L
+    var suggestedVersionCode = 0L
+    val categories = linkedSetOf<String>()
+    val antiFeatures = linkedSetOf<String>()
+    val licenses = mutableListOf<String>()
+    val donates = mutableListOf<Product.Donate>()
+    val releases = mutableListOf<Release>()
+
+    fun build(): Product {
+      return Product(repositoryId, packageName, name, summary, description, "", icon, "",
+        Product.Author(authorName, authorEmail, ""), source, changelog, web, tracker, added, updated,
+        suggestedVersionCode, categories.toList(), antiFeatures.toList(),
+        licenses, donates.sortedWith(DonateComparator), emptyList(), releases)
+    }
+  }
+
+  private class ReleaseBuilder {
+    var version = ""
+    var versionCode = 0L
+    var added = 0L
+    var size = 0L
+    var minSdkVersion = 0
+    var targetSdkVersion = 0
+    var maxSdkVersion = 0
+    var source = ""
+    var release = ""
+    var hash = ""
+    var hashType = ""
+    var signature = ""
+    var obbMain = ""
+    var obbMainHash = ""
+    var obbPatch = ""
+    var obbPatchHash = ""
+    val permissions = linkedSetOf<String>()
+    val features = linkedSetOf<String>()
+    val platforms = linkedSetOf<String>()
+
+    fun build(): Release {
+      val hashType = if (hash.isNotEmpty() && hashType.isEmpty()) "sha256" else hashType
+      val obbMainHashType = if (obbMainHash.isNotEmpty()) "sha256" else ""
+      val obbPatchHashType = if (obbPatchHash.isNotEmpty()) "sha256" else ""
+      return Release(false, version, versionCode, added, size,
+        minSdkVersion, targetSdkVersion, maxSdkVersion, source, release, hash, hashType, signature,
+        obbMain, obbMainHash, obbMainHashType, obbPatch, obbPatchHash, obbPatchHashType,
+        permissions.toList(), features.toList(), platforms.toList(), emptyList())
+    }
+  }
+
+  private val contentBuilder = StringBuilder()
+
+  private var repositoryBuilder: RepositoryBuilder? = RepositoryBuilder()
+  private var productBuilder: ProductBuilder? = null
+  private var releaseBuilder: ReleaseBuilder? = null
+
+  private fun Attributes.get(localName: String): String = getValue("", localName).orEmpty()
+  private fun String.cleanWhiteSpace(): String = replace("\\s".toRegex(), " ")
+
+  override fun startElement(uri: String, localName: String, qName: String, attributes: Attributes) {
+    super.startElement(uri, localName, qName, attributes)
+
+    val repositoryBuilder = repositoryBuilder
+    val productBuilder = productBuilder
+    val releaseBuilder = releaseBuilder
+    contentBuilder.setLength(0)
+
+    when {
+      localName == "repo" -> {
+        if (repositoryBuilder != null) {
+          repositoryBuilder.address = attributes.get("url").cleanWhiteSpace()
+          repositoryBuilder.name = attributes.get("name").cleanWhiteSpace()
+          repositoryBuilder.description = attributes.get("description").cleanWhiteSpace()
+          repositoryBuilder.certificate = attributes.get("pubkey")
+          repositoryBuilder.version = attributes.get("version").toIntOrNull() ?: 0
+          repositoryBuilder.timestamp = (attributes.get("timestamp").toLongOrNull() ?: 0L) * 1000L
+        }
+      }
+      localName == "application" && productBuilder == null -> {
+        this.productBuilder = ProductBuilder(repositoryId, attributes.get("id"))
+      }
+      localName == "package" && productBuilder != null && releaseBuilder == null -> {
+        this.releaseBuilder = ReleaseBuilder()
+      }
+      localName == "hash" && releaseBuilder != null -> {
+        releaseBuilder.hashType = attributes.get("type")
+      }
+      (localName == "uses-permission" || localName.startsWith("uses-permission-")) && releaseBuilder != null -> {
+        val minSdkVersion = if (localName != "uses-permission") {
+          "uses-permission-sdk-(\\d+)".toRegex().matchEntire(localName)
+            ?.destructured?.let { (version) -> version.toIntOrNull() }
+        } else {
+          null
+        } ?: 0
+        val maxSdkVersion = attributes.get("maxSdkVersion").toIntOrNull() ?: Int.MAX_VALUE
+        if (Android.sdk in minSdkVersion .. maxSdkVersion) {
+          releaseBuilder.permissions.add(attributes.get("name"))
+        } else {
+          releaseBuilder.permissions.remove(attributes.get("name"))
+        }
+      }
+    }
+  }
+
+  override fun endElement(uri: String, localName: String, qName: String) {
+    super.endElement(uri, localName, qName)
+
+    val repositoryBuilder = repositoryBuilder
+    val productBuilder = productBuilder
+    val releaseBuilder = releaseBuilder
+    val content = contentBuilder.toString()
+
+    when {
+      localName == "repo" -> {
+        if (repositoryBuilder != null) {
+          val mirrors = (listOf(repositoryBuilder.address) + repositoryBuilder.mirrors)
+            .filter { it.isNotEmpty() }.distinct()
+          callback.onRepository(mirrors, repositoryBuilder.name, repositoryBuilder.description,
+            repositoryBuilder.certificate, repositoryBuilder.version, repositoryBuilder.timestamp)
+          this.repositoryBuilder = null
+        }
+      }
+      localName == "application" && productBuilder != null -> {
+        val product = productBuilder.build()
+        this.productBuilder = null
+        callback.onProduct(product)
+      }
+      localName == "package" && productBuilder != null && releaseBuilder != null -> {
+        productBuilder.releases.add(releaseBuilder.build())
+        this.releaseBuilder = null
+      }
+      repositoryBuilder != null -> {
+        when (localName) {
+          "description" -> repositoryBuilder.description = content.cleanWhiteSpace()
+          "mirror" -> repositoryBuilder.mirrors += content
+        }
+      }
+      productBuilder != null && releaseBuilder != null -> {
+        when (localName) {
+          "version" -> releaseBuilder.version = content
+          "versioncode" -> releaseBuilder.versionCode = content.toLongOrNull() ?: 0L
+          "added" -> releaseBuilder.added = content.parseDate()
+          "size" -> releaseBuilder.size = content.toLongOrNull() ?: 0
+          "sdkver" -> releaseBuilder.minSdkVersion = content.toIntOrNull() ?: 0
+          "targetSdkVersion" -> releaseBuilder.targetSdkVersion = content.toIntOrNull() ?: 0
+          "maxsdkver" -> releaseBuilder.maxSdkVersion = content.toIntOrNull() ?: 0
+          "srcname" -> releaseBuilder.source = content
+          "apkname" -> releaseBuilder.release = content
+          "hash" -> releaseBuilder.hash = content
+          "sig" -> releaseBuilder.signature = content
+          "obbMainFile" -> releaseBuilder.obbMain = content
+          "obbMainFileSha256" -> releaseBuilder.obbMainHash = content
+          "obbPatchFile" -> releaseBuilder.obbPatch = content
+          "obbPatchFileSha256" -> releaseBuilder.obbPatchHash = content
+          "permissions" -> releaseBuilder.permissions += content.split(',').filter { it.isNotEmpty() }
+          "features" -> releaseBuilder.features += content.split(',').filter { it.isNotEmpty() }
+          "nativecode" -> releaseBuilder.platforms += content.split(',').filter { it.isNotEmpty() }
+        }
+      }
+      productBuilder != null -> {
+        when (localName) {
+          "name" -> productBuilder.name = content
+          "summary" -> productBuilder.summary = content
+          "description" -> productBuilder.description = "<p>$content</p>"
+          "desc" -> productBuilder.description = content.replace("\n", "<br/>")
+          "icon" -> productBuilder.icon = validateIcon(content)
+          "author" -> productBuilder.authorName = content
+          "email" -> productBuilder.authorEmail = content
+          "source" -> productBuilder.source = content
+          "changelog" -> productBuilder.changelog = content
+          "web" -> productBuilder.web = content
+          "tracker" -> productBuilder.tracker = content
+          "added" -> productBuilder.added = content.parseDate()
+          "lastupdated" -> productBuilder.updated = content.parseDate()
+          "marketvercode" -> productBuilder.suggestedVersionCode = content.toLongOrNull() ?: 0L
+          "categories" -> productBuilder.categories += content.split(',').filter { it.isNotEmpty() }
+          "antifeatures" -> productBuilder.antiFeatures += content.split(',').filter { it.isNotEmpty() }
+          "license" -> productBuilder.licenses += content.split(',').filter { it.isNotEmpty() }
+          "donate" -> productBuilder.donates += Product.Donate.Regular(content)
+          "bitcoin" -> productBuilder.donates += Product.Donate.Bitcoin(content)
+          "litecoin" -> productBuilder.donates += Product.Donate.Litecoin(content)
+          "flattr" -> productBuilder.donates += Product.Donate.Flattr(content)
+          "liberapay" -> productBuilder.donates += Product.Donate.Liberapay(content)
+          "openCollective" -> productBuilder.donates += Product.Donate.OpenCollective(content)
+        }
+      }
+    }
+  }
+
+  override fun characters(ch: CharArray, start: Int, length: Int) {
+    super.characters(ch, start, length)
+    contentBuilder.append(ch, start, length)
+  }
+}

src/main/kotlin/com/looker/droidify/index/IndexMerger.kt

@@ -0,0 +1,83 @@
+package com.looker.droidify.index
+
+import android.content.ContentValues
+import android.database.sqlite.SQLiteDatabase
+import com.fasterxml.jackson.core.JsonToken
+import com.looker.droidify.entity.Product
+import com.looker.droidify.entity.Release
+import com.looker.droidify.utility.extension.android.*
+import com.looker.droidify.utility.extension.json.*
+import java.io.ByteArrayOutputStream
+import java.io.Closeable
+import java.io.File
+
+class IndexMerger(file: File): Closeable {
+  private val db = SQLiteDatabase.openOrCreateDatabase(file, null)
+
+  init {
+    db.execWithResult("PRAGMA synchronous = OFF")
+    db.execWithResult("PRAGMA journal_mode = OFF")
+    db.execSQL("CREATE TABLE product (package_name TEXT PRIMARY KEY, description TEXT NOT NULL, data BLOB NOT NULL)")
+    db.execSQL("CREATE TABLE releases (package_name TEXT PRIMARY KEY, data BLOB NOT NULL)")
+    db.beginTransaction()
+  }
+
+  fun addProducts(products: List<Product>) {
+    for (product in products) {
+      val outputStream = ByteArrayOutputStream()
+      Json.factory.createGenerator(outputStream).use { it.writeDictionary(product::serialize) }
+      db.insert("product", null, ContentValues().apply {
+        put("package_name", product.packageName)
+        put("description", product.description)
+        put("data", outputStream.toByteArray())
+      })
+    }
+  }
+
+  fun addReleases(pairs: List<Pair<String, List<Release>>>) {
+    for (pair in pairs) {
+      val (packageName, releases) = pair
+      val outputStream = ByteArrayOutputStream()
+      Json.factory.createGenerator(outputStream).use {
+        it.writeStartArray()
+        for (release in releases) {
+          it.writeDictionary(release::serialize)
+        }
+        it.writeEndArray()
+      }
+      db.insert("releases", null, ContentValues().apply {
+        put("package_name", packageName)
+        put("data", outputStream.toByteArray())
+      })
+    }
+  }
+
+  private fun closeTransaction() {
+    if (db.inTransaction()) {
+      db.setTransactionSuccessful()
+      db.endTransaction()
+    }
+  }
+
+  fun forEach(repositoryId: Long, windowSize: Int, callback: (List<Product>, Int) -> Unit) {
+    closeTransaction()
+    db.rawQuery("""SELECT product.description, product.data AS pd, releases.data AS rd FROM product
+      LEFT JOIN releases ON product.package_name = releases.package_name""", null)
+      ?.use { it.asSequence().map {
+        val description = it.getString(0)
+        val product = Json.factory.createParser(it.getBlob(1)).use {
+          it.nextToken()
+          Product.deserialize(repositoryId, description, it)
+        }
+        val releases = it.getBlob(2)?.let { Json.factory.createParser(it).use {
+          it.nextToken()
+          it.collectNotNull(JsonToken.START_OBJECT, Release.Companion::deserialize)
+        } }.orEmpty()
+        product.copy(releases = releases)
+      }.windowed(windowSize, windowSize, true).forEach { products -> callback(products, it.count) } }
+  }
+
+  override fun close() {
+    db.use { closeTransaction() }
+  }
+}

src/main/kotlin/com/looker/droidify/index/IndexV1Parser.kt

@@ -0,0 +1,260 @@
+package com.looker.droidify.index
+
+import com.fasterxml.jackson.core.JsonParser
+import com.fasterxml.jackson.core.JsonToken
+import com.looker.droidify.entity.Product
+import com.looker.droidify.entity.Release
+import com.looker.droidify.utility.extension.android.*
+import com.looker.droidify.utility.extension.json.*
+import com.looker.droidify.utility.extension.text.*
+import java.io.InputStream
+
+object IndexV1Parser {
+  interface Callback {
+    fun onRepository(mirrors: List<String>, name: String, description: String, version: Int, timestamp: Long)
+    fun onProduct(product: Product)
+    fun onReleases(packageName: String, releases: List<Release>)
+  }
+
+  private class Screenshots(val phone: List<String>, val smallTablet: List<String>, val largeTablet: List<String>)
+  private class Localized(val name: String, val summary: String, val description: String,
+    val whatsNew: String, val metadataIcon: String, val screenshots: Screenshots?)
+
+  private fun <T> Map<String, Localized>.getAndCall(key: String, callback: (String, Localized) -> T?): T? {
+    return this[key]?.let { callback(key, it) }
+  }
+
+  private fun <T> Map<String, Localized>.find(callback: (String, Localized) -> T?): T? {
+    return getAndCall("en-US", callback) ?: getAndCall("en_US", callback) ?: getAndCall("en", callback)
+  }
+
+  private fun Map<String, Localized>.findString(fallback: String, callback: (Localized) -> String): String {
+    return (find { _, localized -> callback(localized).nullIfEmpty() } ?: fallback).trim()
+  }
+
+  fun parse(repositoryId: Long, inputStream: InputStream, callback: Callback) {
+    val jsonParser = Json.factory.createParser(inputStream)
+    if (jsonParser.nextToken() != JsonToken.START_OBJECT) {
+      jsonParser.illegal()
+    } else {
+      jsonParser.forEachKey {
+        when {
+          it.dictionary("repo") -> {
+            var address = ""
+            var mirrors = emptyList<String>()
+            var name = ""
+            var description = ""
+            var version = 0
+            var timestamp = 0L
+            forEachKey {
+              when {
+                it.string("address") -> address = valueAsString
+                it.array("mirrors") -> mirrors = collectDistinctNotEmptyStrings()
+                it.string("name") -> name = valueAsString
+                it.string("description") -> description = valueAsString
+                it.number("version") -> version = valueAsInt
+                it.number("timestamp") -> timestamp = valueAsLong
+                else -> skipChildren()
+              }
+            }
+            val realMirrors = ((if (address.isNotEmpty()) listOf(address) else emptyList()) + mirrors).distinct()
+            callback.onRepository(realMirrors, name, description, version, timestamp)
+          }
+          it.array("apps") -> forEach(JsonToken.START_OBJECT) {
+            val product = parseProduct(repositoryId)
+            callback.onProduct(product)
+          }
+          it.dictionary("packages") -> forEachKey {
+            if (it.token == JsonToken.START_ARRAY) {
+              val packageName = it.key
+              val releases = collectNotNull(JsonToken.START_OBJECT) { parseRelease() }
+              callback.onReleases(packageName, releases)
+            } else {
+              skipChildren()
+            }
+          }
+          else -> skipChildren()
+        }
+      }
+    }
+  }
+
+  private fun JsonParser.parseProduct(repositoryId: Long): Product {
+    var packageName = ""
+    var nameFallback = ""
+    var summaryFallback = ""
+    var descriptionFallback = ""
+    var icon = ""
+    var authorName = ""
+    var authorEmail = ""
+    var authorWeb = ""
+    var source = ""
+    var changelog = ""
+    var web = ""
+    var tracker = ""
+    var added = 0L
+    var updated = 0L
+    var suggestedVersionCode = 0L
+    var categories = emptyList<String>()
+    var antiFeatures = emptyList<String>()
+    val licenses = mutableListOf<String>()
+    val donates = mutableListOf<Product.Donate>()
+    val localizedMap = mutableMapOf<String, Localized>()
+    forEachKey {
+      when {
+        it.string("packageName") -> packageName = valueAsString
+        it.string("name") -> nameFallback = valueAsString
+        it.string("summary") -> summaryFallback = valueAsString
+        it.string("description") -> descriptionFallback = valueAsString
+        it.string("icon") -> icon = IndexHandler.validateIcon(valueAsString)
+        it.string("authorName") -> authorName = valueAsString
+        it.string("authorEmail") -> authorEmail = valueAsString
+        it.string("authorWebSite") -> authorWeb = valueAsString
+        it.string("sourceCode") -> source = valueAsString
+        it.string("changelog") -> changelog = valueAsString
+        it.string("webSite") -> web = valueAsString
+        it.string("issueTracker") -> tracker = valueAsString
+        it.number("added") -> added = valueAsLong
+        it.number("lastUpdated") -> updated = valueAsLong
+        it.string("suggestedVersionCode") -> suggestedVersionCode = valueAsString.toLongOrNull() ?: 0L
+        it.array("categories") -> categories = collectDistinctNotEmptyStrings()
+        it.array("antiFeatures") -> antiFeatures = collectDistinctNotEmptyStrings()
+        it.string("license") -> licenses += valueAsString.split(',').filter { it.isNotEmpty() }
+        it.string("donate") -> donates += Product.Donate.Regular(valueAsString)
+        it.string("bitcoin") -> donates += Product.Donate.Bitcoin(valueAsString)
+        it.string("flattrID") -> donates += Product.Donate.Flattr(valueAsString)
+        it.string("liberapayID") -> donates += Product.Donate.Liberapay(valueAsString)
+        it.string("openCollective") -> donates += Product.Donate.OpenCollective(valueAsString)
+        it.dictionary("localized") -> forEachKey {
+          if (it.token == JsonToken.START_OBJECT) {
+            val locale = it.key
+            var name = ""
+            var summary = ""
+            var description = ""
+            var whatsNew = ""
+            var metadataIcon = ""
+            var phone = emptyList<String>()
+            var smallTablet = emptyList<String>()
+            var largeTablet = emptyList<String>()
+            forEachKey {
+              when {
+                it.string("name") -> name = valueAsString
+                it.string("summary") -> summary = valueAsString
+                it.string("description") -> description = valueAsString
+                it.string("whatsNew") -> whatsNew = valueAsString
+                it.string("icon") -> metadataIcon = valueAsString
+                it.array("phoneScreenshots") -> phone = collectDistinctNotEmptyStrings()
+                it.array("sevenInchScreenshots") -> smallTablet = collectDistinctNotEmptyStrings()
+                it.array("tenInchScreenshots") -> largeTablet = collectDistinctNotEmptyStrings()
+                else -> skipChildren()
+              }
+            }
+            val screenshots = if (sequenceOf(phone, smallTablet, largeTablet).any { it.isNotEmpty() })
+              Screenshots(phone, smallTablet, largeTablet) else null
+            localizedMap[locale] = Localized(name, summary, description, whatsNew,
+              metadataIcon.nullIfEmpty()?.let { "$locale/$it" }.orEmpty(), screenshots)
+          } else {
+            skipChildren()
+          }
+        }
+        else -> skipChildren()
+      }
+    }
+    val name = localizedMap.findString(nameFallback) { it.name }
+    val summary = localizedMap.findString(summaryFallback) { it.summary }
+    val description = localizedMap.findString(descriptionFallback) { it.description }.replace("\n", "<br/>")
+    val whatsNew = localizedMap.findString("") { it.whatsNew }.replace("\n", "<br/>")
+    val metadataIcon = localizedMap.findString("") { it.metadataIcon }
+    val screenshotPairs = localizedMap.find { key, localized -> localized.screenshots?.let { Pair(key, it) } }
+    val screenshots = screenshotPairs
+      ?.let { (key, screenshots) -> screenshots.phone.asSequence()
+        .map { Product.Screenshot(key, Product.Screenshot.Type.PHONE, it) } +
+        screenshots.smallTablet.asSequence()
+          .map { Product.Screenshot(key, Product.Screenshot.Type.SMALL_TABLET, it) } +
+        screenshots.largeTablet.asSequence()
+          .map { Product.Screenshot(key, Product.Screenshot.Type.LARGE_TABLET, it) } }
+      .orEmpty().toList()
+    return Product(repositoryId, packageName, name, summary, description, whatsNew, icon, metadataIcon,
+      Product.Author(authorName, authorEmail, authorWeb), source, changelog, web, tracker, added, updated,
+      suggestedVersionCode, categories, antiFeatures, licenses,
+      donates.sortedWith(IndexHandler.DonateComparator), screenshots, emptyList())
+  }
+
+  private fun JsonParser.parseRelease(): Release {
+    var version = ""
+    var versionCode = 0L
+    var added = 0L
+    var size = 0L
+    var minSdkVersion = 0
+    var targetSdkVersion = 0
+    var maxSdkVersion = 0
+    var source = ""
+    var release = ""
+    var hash = ""
+    var hashTypeCandidate = ""
+    var signature = ""
+    var obbMain = ""
+    var obbMainHash = ""
+    var obbPatch = ""
+    var obbPatchHash = ""
+    val permissions = linkedSetOf<String>()
+    var features = emptyList<String>()
+    var platforms = emptyList<String>()
+    forEachKey {
+      when {
+        it.string("versionName") -> version = valueAsString
+        it.number("versionCode") -> versionCode = valueAsLong
+        it.number("added") -> added = valueAsLong
+        it.number("size") -> size = valueAsLong
+        it.number("minSdkVersion") -> minSdkVersion = valueAsInt
+        it.number("targetSdkVersion") -> targetSdkVersion = valueAsInt
+        it.number("maxSdkVersion") -> maxSdkVersion = valueAsInt
+        it.string("srcname") -> source = valueAsString
+        it.string("apkName") -> release = valueAsString
+        it.string("hash") -> hash = valueAsString
+        it.string("hashType") -> hashTypeCandidate = valueAsString
+        it.string("sig") -> signature = valueAsString
+        it.string("obbMainFile") -> obbMain = valueAsString
+        it.string("obbMainFileSha256") -> obbMainHash = valueAsString
+        it.string("obbPatchFile") -> obbPatch = valueAsString
+        it.string("obbPatchFileSha256") -> obbPatchHash = valueAsString
+        it.array("uses-permission") -> collectPermissions(permissions, 0)
+        it.array("uses-permission-sdk-23") -> collectPermissions(permissions, 23)
+        it.array("features") -> features = collectDistinctNotEmptyStrings()
+        it.array("nativecode") -> platforms = collectDistinctNotEmptyStrings()
+        else -> skipChildren()
+      }
+    }
+    val hashType = if (hash.isNotEmpty() && hashTypeCandidate.isEmpty()) "sha256" else hashTypeCandidate
+    val obbMainHashType = if (obbMainHash.isNotEmpty()) "sha256" else ""
+    val obbPatchHashType = if (obbPatchHash.isNotEmpty()) "sha256" else ""
+    return Release(false, version, versionCode, added, size,
+      minSdkVersion, targetSdkVersion, maxSdkVersion, source, release, hash, hashType, signature,
+      obbMain, obbMainHash, obbMainHashType, obbPatch, obbPatchHash, obbPatchHashType,
+      permissions.toList(), features, platforms, emptyList())
+  }
+
+  private fun JsonParser.collectPermissions(permissions: LinkedHashSet<String>, minSdk: Int) {
+    forEach(JsonToken.START_ARRAY) {
+      val firstToken = nextToken()
+      val permission = if (firstToken == JsonToken.VALUE_STRING) valueAsString else ""
+      if (firstToken != JsonToken.END_ARRAY) {
+        val secondToken = nextToken()
+        val maxSdk = if (secondToken == JsonToken.VALUE_NUMBER_INT) valueAsInt else 0
+        if (permission.isNotEmpty() && Android.sdk >= minSdk && (maxSdk <= 0 || Android.sdk <= maxSdk)) {
+          permissions.add(permission)
+        }
+        if (secondToken != JsonToken.END_ARRAY) {
+          while (true) {
+            val token = nextToken()
+            if (token == JsonToken.END_ARRAY) {
+              break
+            } else if (token.isStructStart) {
+              skipChildren()
+            }
+          }
+        }
+      }
+    }
+  }
+}

src/main/kotlin/com/looker/droidify/index/RepositoryUpdater.kt

@@ -0,0 +1,345 @@
+package com.looker.droidify.index
+
+import android.content.Context
+import android.net.Uri
+import io.reactivex.rxjava3.core.Observable
+import io.reactivex.rxjava3.core.Single
+import io.reactivex.rxjava3.schedulers.Schedulers
+import com.looker.droidify.content.Cache
+import com.looker.droidify.database.Database
+import com.looker.droidify.entity.Product
+import com.looker.droidify.entity.Release
+import com.looker.droidify.entity.Repository
+import com.looker.droidify.network.Downloader
+import com.looker.droidify.utility.ProgressInputStream
+import com.looker.droidify.utility.RxUtils
+import com.looker.droidify.utility.Utils
+import com.looker.droidify.utility.extension.android.*
+import com.looker.droidify.utility.extension.text.*
+import org.xml.sax.InputSource
+import java.io.File
+import java.security.cert.X509Certificate
+import java.util.Locale
+import java.util.jar.JarEntry
+import java.util.jar.JarFile
+import javax.xml.parsers.SAXParserFactory
+
+object RepositoryUpdater {
+  enum class Stage {
+    DOWNLOAD, PROCESS, MERGE, COMMIT
+  }
+
+  private enum class IndexType(val jarName: String, val contentName: String, val certificateFromIndex: Boolean) {
+    INDEX("index.jar", "index.xml", true),
+    INDEX_V1("index-v1.jar", "index-v1.json", false)
+  }
+
+  enum class ErrorType {
+    NETWORK, HTTP, VALIDATION, PARSING
+  }
+
+  class UpdateException: Exception {
+    val errorType: ErrorType
+
+    constructor(errorType: ErrorType, message: String): super(message) {
+      this.errorType = errorType
+    }
+
+    constructor(errorType: ErrorType, message: String, cause: Exception): super(message, cause) {
+      this.errorType = errorType
+    }
+  }
+
+  private lateinit var context: Context
+  private val updaterLock = Any()
+  private val cleanupLock = Any()
+
+  fun init(context: Context) {
+    this.context = context
+
+    var lastDisabled = setOf<Long>()
+    Observable.just(Unit)
+      .concatWith(Database.observable(Database.Subject.Repositories))
+      .observeOn(Schedulers.io())
+      .flatMapSingle { RxUtils.querySingle { Database.RepositoryAdapter.getAllDisabledDeleted(it) } }
+      .forEach {
+        val newDisabled = it.asSequence().filter { !it.second }.map { it.first }.toSet()
+        val disabled = newDisabled - lastDisabled
+        lastDisabled = newDisabled
+        val deleted = it.asSequence().filter { it.second }.map { it.first }.toSet()
+        if (disabled.isNotEmpty() || deleted.isNotEmpty()) {
+          val pairs = (disabled.asSequence().map { Pair(it, false) } +
+            deleted.asSequence().map { Pair(it, true) }).toSet()
+          synchronized(cleanupLock) { Database.RepositoryAdapter.cleanup(pairs) }
+        }
+      }
+  }
+
+  fun await() {
+    synchronized(updaterLock) { }
+  }
+
+  fun update(repository: Repository, unstable: Boolean,
+    callback: (Stage, Long, Long?) -> Unit): Single<Boolean> {
+    return update(repository, listOf(IndexType.INDEX_V1, IndexType.INDEX), unstable, callback)
+  }
+
+  private fun update(repository: Repository, indexTypes: List<IndexType>, unstable: Boolean,
+    callback: (Stage, Long, Long?) -> Unit): Single<Boolean> {
+    val indexType = indexTypes[0]
+    return downloadIndex(repository, indexType, callback)
+      .flatMap { (result, file) ->
+        when {
+          result.isNotChanged -> {
+            file.delete()
+            Single.just(false)
+          }
+          !result.success -> {
+            file.delete()
+            if (result.code == 404 && indexTypes.isNotEmpty()) {
+              update(repository, indexTypes.subList(1, indexTypes.size), unstable, callback)
+            } else {
+              Single.error(UpdateException(ErrorType.HTTP, "Invalid response: HTTP ${result.code}"))
+            }
+          }
+          else -> {
+            RxUtils.managedSingle { processFile(repository, indexType, unstable,
+              file, result.lastModified, result.entityTag, callback) }
+          }
+        }
+      }
+  }
+
+  private fun downloadIndex(repository: Repository, indexType: IndexType,
+    callback: (Stage, Long, Long?) -> Unit): Single<Pair<Downloader.Result, File>> {
+    return Single.just(Unit)
+      .map { Cache.getTemporaryFile(context) }
+      .flatMap { file -> Downloader
+        .download(Uri.parse(repository.address).buildUpon()
+          .appendPath(indexType.jarName).build().toString(), file, repository.lastModified, repository.entityTag,
+          repository.authentication) { read, total -> callback(Stage.DOWNLOAD, read, total) }
+        .subscribeOn(Schedulers.io())
+        .map { Pair(it, file) }
+        .onErrorResumeNext {
+          file.delete()
+          when (it) {
+            is InterruptedException, is RuntimeException, is Error -> Single.error(it)
+            is Exception -> Single.error(UpdateException(ErrorType.NETWORK, "Network error", it))
+            else -> Single.error(it)
+          }
+        } }
+  }
+
+  private fun processFile(repository: Repository, indexType: IndexType, unstable: Boolean,
+    file: File, lastModified: String, entityTag: String, callback: (Stage, Long, Long?) -> Unit): Boolean {
+    var rollback = true
+    return synchronized(updaterLock) {
+      try {
+        val jarFile = JarFile(file, true)
+        val indexEntry = jarFile.getEntry(indexType.contentName) as JarEntry
+        val total = indexEntry.size
+        Database.UpdaterAdapter.createTemporaryTable()
+        val features = context.packageManager.systemAvailableFeatures
+          .asSequence().map { it.name }.toSet() + setOf("android.hardware.touchscreen")
+
+        val (changedRepository, certificateFromIndex) = when (indexType) {
+          IndexType.INDEX -> {
+            val factory = SAXParserFactory.newInstance()
+            factory.isNamespaceAware = true
+            val parser = factory.newSAXParser()
+            val reader = parser.xmlReader
+            var changedRepository: Repository? = null
+            var certificateFromIndex: String? = null
+            val products = mutableListOf<Product>()
+
+            reader.contentHandler = IndexHandler(repository.id, object: IndexHandler.Callback {
+              override fun onRepository(mirrors: List<String>, name: String, description: String,
+                certificate: String, version: Int, timestamp: Long) {
+                changedRepository = repository.update(mirrors, name, description, version,
+                  lastModified, entityTag, timestamp)
+                certificateFromIndex = certificate.toLowerCase(Locale.US)
+              }
+
+              override fun onProduct(product: Product) {
+                if (Thread.interrupted()) {
+                  throw InterruptedException()
+                }
+                products += transformProduct(product, features, unstable)
+                if (products.size >= 50) {
+                  Database.UpdaterAdapter.putTemporary(products)
+                  products.clear()
+                }
+              }
+            })
+
+            ProgressInputStream(jarFile.getInputStream(indexEntry)) { callback(Stage.PROCESS, it, total) }
+              .use { reader.parse(InputSource(it)) }
+            if (Thread.interrupted()) {
+              throw InterruptedException()
+            }
+            if (products.isNotEmpty()) {
+              Database.UpdaterAdapter.putTemporary(products)
+              products.clear()
+            }
+            Pair(changedRepository, certificateFromIndex)
+          }
+          IndexType.INDEX_V1 -> {
+            var changedRepository: Repository? = null
+
+            val mergerFile = Cache.getTemporaryFile(context)
+            try {
+              val unmergedProducts = mutableListOf<Product>()
+              val unmergedReleases = mutableListOf<Pair<String, List<Release>>>()
+              IndexMerger(mergerFile).use { indexMerger ->
+                ProgressInputStream(jarFile.getInputStream(indexEntry)) { callback(Stage.PROCESS, it, total) }.use {
+                  IndexV1Parser.parse(repository.id, it, object: IndexV1Parser.Callback {
+                    override fun onRepository(mirrors: List<String>, name: String, description: String,
+                      version: Int, timestamp: Long) {
+                      changedRepository = repository.update(mirrors, name, description, version,
+                        lastModified, entityTag, timestamp)
+                    }
+
+                    override fun onProduct(product: Product) {
+                      if (Thread.interrupted()) {
+                        throw InterruptedException()
+                      }
+                      unmergedProducts += product
+                      if (unmergedProducts.size >= 50) {
+                        indexMerger.addProducts(unmergedProducts)
+                        unmergedProducts.clear()
+                      }
+                    }
+
+                    override fun onReleases(packageName: String, releases: List<Release>) {
+                      if (Thread.interrupted()) {
+                        throw InterruptedException()
+                      }
+                      unmergedReleases += Pair(packageName, releases)
+                      if (unmergedReleases.size >= 50) {
+                        indexMerger.addReleases(unmergedReleases)
+                        unmergedReleases.clear()
+                      }
+                    }
+                  })
+
+                  if (Thread.interrupted()) {
+                    throw InterruptedException()
+                  }
+                  if (unmergedProducts.isNotEmpty()) {
+                    indexMerger.addProducts(unmergedProducts)
+                    unmergedProducts.clear()
+                  }
+                  if (unmergedReleases.isNotEmpty()) {
+                    indexMerger.addReleases(unmergedReleases)
+                    unmergedReleases.clear()
+                  }
+                  var progress = 0
+                  indexMerger.forEach(repository.id, 50) { products, totalCount ->
+                    if (Thread.interrupted()) {
+                      throw InterruptedException()
+                    }
+                    progress += products.size
+                    callback(Stage.MERGE, progress.toLong(), totalCount.toLong())
+                    Database.UpdaterAdapter.putTemporary(products
+                      .map { transformProduct(it, features, unstable) })
+                  }
+                }
+              }
+            } finally {
+              mergerFile.delete()
+            }
+            Pair(changedRepository, null)
+          }
+        }
+
+        val workRepository = changedRepository ?: repository
+        if (workRepository.timestamp < repository.timestamp) {
+          throw UpdateException(ErrorType.VALIDATION, "New index is older than current index: " +
+            "${workRepository.timestamp} < ${repository.timestamp}")
+        } else {
+          val fingerprint = run {
+            val certificateFromJar = run {
+              val codeSigners = indexEntry.codeSigners
+              if (codeSigners == null || codeSigners.size != 1) {
+                throw UpdateException(ErrorType.VALIDATION, "index.jar must be signed by a single code signer")
+              } else {
+                val certificates = codeSigners[0].signerCertPath?.certificates.orEmpty()
+                if (certificates.size != 1) {
+                  throw UpdateException(ErrorType.VALIDATION, "index.jar code signer should have only one certificate")
+                } else {
+                  certificates[0] as X509Certificate
+                }
+              }
+            }
+            val fingerprintFromJar = Utils.calculateFingerprint(certificateFromJar)
+            if (indexType.certificateFromIndex) {
+              val fingerprintFromIndex = certificateFromIndex?.unhex()?.let(Utils::calculateFingerprint)
+              if (fingerprintFromIndex == null || fingerprintFromJar != fingerprintFromIndex) {
+                throw UpdateException(ErrorType.VALIDATION, "index.xml contains invalid public key")
+              }
+              fingerprintFromIndex
+            } else {
+              fingerprintFromJar
+            }
+          }
+
+          val commitRepository = if (workRepository.fingerprint != fingerprint) {
+            if (workRepository.fingerprint.isEmpty()) {
+              workRepository.copy(fingerprint = fingerprint)
+            } else {
+              throw UpdateException(ErrorType.VALIDATION, "Certificate fingerprints do not match")
+            }
+          } else {
+            workRepository
+          }
+          if (Thread.interrupted()) {
+            throw InterruptedException()
+          }
+          callback(Stage.COMMIT, 0, null)
+          synchronized(cleanupLock) { Database.UpdaterAdapter.finishTemporary(commitRepository, true) }
+          rollback = false
+          true
+        }
+      } catch (e: Exception) {
+        throw when (e) {
+          is UpdateException, is InterruptedException -> e
+          else -> UpdateException(ErrorType.PARSING, "Error parsing index", e)
+        }
+      } finally {
+        file.delete()
+        if (rollback) {
+          Database.UpdaterAdapter.finishTemporary(repository, false)
+        }
+      }
+    }
+  }
+
+  private fun transformProduct(product: Product, features: Set<String>, unstable: Boolean): Product {
+    val releasePairs = product.releases.distinctBy { it.identifier }.sortedByDescending { it.versionCode }.map {
+      val incompatibilities = mutableListOf<Release.Incompatibility>()
+      if (it.minSdkVersion > 0 && Android.sdk < it.minSdkVersion) {
+        incompatibilities += Release.Incompatibility.MinSdk
+      }
+      if (it.maxSdkVersion > 0 && Android.sdk > it.maxSdkVersion) {
+        incompatibilities += Release.Incompatibility.MaxSdk
+      }
+      if (it.platforms.isNotEmpty() && it.platforms.intersect(Android.platforms).isEmpty()) {
+        incompatibilities += Release.Incompatibility.Platform
+      }
+      incompatibilities += (it.features - features).sorted().map { Release.Incompatibility.Feature(it) }
+      Pair(it, incompatibilities as List<Release.Incompatibility>)
+    }.toMutableList()
+
+    val predicate: (Release) -> Boolean = { unstable || product.suggestedVersionCode <= 0 ||
+      it.versionCode <= product.suggestedVersionCode }
+    val firstCompatibleReleaseIndex = releasePairs.indexOfFirst { it.second.isEmpty() && predicate(it.first) }
+    val firstReleaseIndex = if (firstCompatibleReleaseIndex >= 0) firstCompatibleReleaseIndex else
+      releasePairs.indexOfFirst { predicate(it.first) }
+    val firstSelected = if (firstReleaseIndex >= 0) releasePairs[firstReleaseIndex] else null
+
+    val releases = releasePairs.map { (release, incompatibilities) -> release
+      .copy(incompatibilities = incompatibilities, selected = firstSelected
+        ?.let { it.first.versionCode == release.versionCode && it.second == incompatibilities } == true) }
+    return product.copy(releases = releases)
+  }
+}