avior/Neo-Store
1
0
Fork 0
mirror of https://github.com/Aviortheking/Neo-Store.git synced 2025-06-16 04:19:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reformated all the code

Browse Source
This commit is contained in:
Mohit
2021-06-08 21:18:44 +05:30
parent 74e8287cf1
commit 29ef88853d
50 changed files with 6043 additions and 4857 deletions
Download Patch File Download Diff File Expand all files Collapse all files

567
src/main/kotlin/com/looker/droidify/index/IndexHandler.kt
View File

@ -2,264 +2,335 @@ package com.looker.droidify.index
import com.looker.droidify.entity.Product
import com.looker.droidify.entity.Release
import com.looker.droidify.utility.extension.android.*
import com.looker.droidify.utility.extension.android.Android
import org.xml.sax.Attributes
import org.xml.sax.helpers.DefaultHandler
import java.text.SimpleDateFormat
import java.util.Locale
import java.util.TimeZone
import java.util.*
class IndexHandler(private val repositoryId: Long, private val callback: Callback): DefaultHandler() {
companion object {
private val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US)
.apply { timeZone = TimeZone.getTimeZone("UTC") }
class IndexHandler(private val repositoryId: Long, private val callback: Callback) :
DefaultHandler() {
companion object {
private val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US)
.apply { timeZone = TimeZone.getTimeZone("UTC") }
private fun String.parseDate(): Long {
return try {
dateFormat.parse(this)?.time ?: 0L
} catch (e: Exception) {
0L
}
}
internal fun validateIcon(icon: String): String {
return if (icon.endsWith(".xml")) "" else icon
}
}
interface Callback {
fun onRepository(mirrors: List<String>, name: String, description: String,
certificate: String, version: Int, timestamp: Long)
fun onProduct(product: Product)
}
internal object DonateComparator: Comparator<Product.Donate> {
private val classes = listOf(Product.Donate.Regular::class, Product.Donate.Bitcoin::class,
Product.Donate.Litecoin::class, Product.Donate.Flattr::class, Product.Donate.Liberapay::class,
Product.Donate.OpenCollective::class)
override fun compare(donate1: Product.Donate, donate2: Product.Donate): Int {
val index1 = classes.indexOf(donate1::class)
val index2 = classes.indexOf(donate2::class)
return when {
index1 >= 0 && index2 == -1 -> -1
index2 >= 0 && index1 == -1 -> 1
else -> index1.compareTo(index2)
}
}
}
private class RepositoryBuilder {
var address = ""
val mirrors = mutableListOf<String>()
var name = ""
var description = ""
var certificate = ""
var version = -1
var timestamp = 0L
}
private class ProductBuilder(val repositoryId: Long, val packageName: String) {
var name = ""
var summary = ""
var description = ""
var icon = ""
var authorName = ""
var authorEmail = ""
var source = ""
var changelog = ""
var web = ""
var tracker = ""
var added = 0L
var updated = 0L
var suggestedVersionCode = 0L
val categories = linkedSetOf<String>()
val antiFeatures = linkedSetOf<String>()
val licenses = mutableListOf<String>()
val donates = mutableListOf<Product.Donate>()
val releases = mutableListOf<Release>()
fun build(): Product {
return Product(repositoryId, packageName, name, summary, description, "", icon, "",
Product.Author(authorName, authorEmail, ""), source, changelog, web, tracker, added, updated,
suggestedVersionCode, categories.toList(), antiFeatures.toList(),
licenses, donates.sortedWith(DonateComparator), emptyList(), releases)
}
}
private class ReleaseBuilder {
var version = ""
var versionCode = 0L
var added = 0L
var size = 0L
var minSdkVersion = 0
var targetSdkVersion = 0
var maxSdkVersion = 0
var source = ""
var release = ""
var hash = ""
var hashType = ""
var signature = ""
var obbMain = ""
var obbMainHash = ""
var obbPatch = ""
var obbPatchHash = ""
val permissions = linkedSetOf<String>()
val features = linkedSetOf<String>()
val platforms = linkedSetOf<String>()
fun build(): Release {
val hashType = if (hash.isNotEmpty() && hashType.isEmpty()) "sha256" else hashType
val obbMainHashType = if (obbMainHash.isNotEmpty()) "sha256" else ""
val obbPatchHashType = if (obbPatchHash.isNotEmpty()) "sha256" else ""
return Release(false, version, versionCode, added, size,
minSdkVersion, targetSdkVersion, maxSdkVersion, source, release, hash, hashType, signature,
obbMain, obbMainHash, obbMainHashType, obbPatch, obbPatchHash, obbPatchHashType,
permissions.toList(), features.toList(), platforms.toList(), emptyList())
}
}
private val contentBuilder = StringBuilder()
private var repositoryBuilder: RepositoryBuilder? = RepositoryBuilder()
private var productBuilder: ProductBuilder? = null
private var releaseBuilder: ReleaseBuilder? = null
private fun Attributes.get(localName: String): String = getValue("", localName).orEmpty()
private fun String.cleanWhiteSpace(): String = replace("\\s".toRegex(), " ")
override fun startElement(uri: String, localName: String, qName: String, attributes: Attributes) {
super.startElement(uri, localName, qName, attributes)
val repositoryBuilder = repositoryBuilder
val productBuilder = productBuilder
val releaseBuilder = releaseBuilder
contentBuilder.setLength(0)
when {
localName == "repo" -> {
if (repositoryBuilder != null) {
repositoryBuilder.address = attributes.get("url").cleanWhiteSpace()
repositoryBuilder.name = attributes.get("name").cleanWhiteSpace()
repositoryBuilder.description = attributes.get("description").cleanWhiteSpace()
repositoryBuilder.certificate = attributes.get("pubkey")
repositoryBuilder.version = attributes.get("version").toIntOrNull() ?: 0
repositoryBuilder.timestamp = (attributes.get("timestamp").toLongOrNull() ?: 0L) * 1000L
private fun String.parseDate(): Long {
return try {
dateFormat.parse(this)?.time ?: 0L
} catch (e: Exception) {
0L
}
}
}
localName == "application" && productBuilder == null -> {
this.productBuilder = ProductBuilder(repositoryId, attributes.get("id"))
}
localName == "package" && productBuilder != null && releaseBuilder == null -> {
this.releaseBuilder = ReleaseBuilder()
}
localName == "hash" && releaseBuilder != null -> {
releaseBuilder.hashType = attributes.get("type")
}
(localName == "uses-permission" || localName.startsWith("uses-permission-")) && releaseBuilder != null -> {
val minSdkVersion = if (localName != "uses-permission") {
"uses-permission-sdk-(\\d+)".toRegex().matchEntire(localName)
?.destructured?.let { (version) -> version.toIntOrNull() }
} else {
null
} ?: 0
val maxSdkVersion = attributes.get("maxSdkVersion").toIntOrNull() ?: Int.MAX_VALUE
if (Android.sdk in minSdkVersion .. maxSdkVersion) {
releaseBuilder.permissions.add(attributes.get("name"))
} else {
releaseBuilder.permissions.remove(attributes.get("name"))
internal fun validateIcon(icon: String): String {
return if (icon.endsWith(".xml")) "" else icon
}
}
}
}
override fun endElement(uri: String, localName: String, qName: String) {
super.endElement(uri, localName, qName)
interface Callback {
fun onRepository(
mirrors: List<String>, name: String, description: String,
certificate: String, version: Int, timestamp: Long
)
val repositoryBuilder = repositoryBuilder
val productBuilder = productBuilder
val releaseBuilder = releaseBuilder
val content = contentBuilder.toString()
when {
localName == "repo" -> {
if (repositoryBuilder != null) {
val mirrors = (listOf(repositoryBuilder.address) + repositoryBuilder.mirrors)
.filter { it.isNotEmpty() }.distinct()
callback.onRepository(mirrors, repositoryBuilder.name, repositoryBuilder.description,
repositoryBuilder.certificate, repositoryBuilder.version, repositoryBuilder.timestamp)
this.repositoryBuilder = null
}
}
localName == "application" && productBuilder != null -> {
val product = productBuilder.build()
this.productBuilder = null
callback.onProduct(product)
}
localName == "package" && productBuilder != null && releaseBuilder != null -> {
productBuilder.releases.add(releaseBuilder.build())
this.releaseBuilder = null
}
repositoryBuilder != null -> {
when (localName) {
"description" -> repositoryBuilder.description = content.cleanWhiteSpace()
"mirror" -> repositoryBuilder.mirrors += content
}
}
productBuilder != null && releaseBuilder != null -> {
when (localName) {
"version" -> releaseBuilder.version = content
"versioncode" -> releaseBuilder.versionCode = content.toLongOrNull() ?: 0L
"added" -> releaseBuilder.added = content.parseDate()
"size" -> releaseBuilder.size = content.toLongOrNull() ?: 0
"sdkver" -> releaseBuilder.minSdkVersion = content.toIntOrNull() ?: 0
"targetSdkVersion" -> releaseBuilder.targetSdkVersion = content.toIntOrNull() ?: 0
"maxsdkver" -> releaseBuilder.maxSdkVersion = content.toIntOrNull() ?: 0
"srcname" -> releaseBuilder.source = content
"apkname" -> releaseBuilder.release = content
"hash" -> releaseBuilder.hash = content
"sig" -> releaseBuilder.signature = content
"obbMainFile" -> releaseBuilder.obbMain = content
"obbMainFileSha256" -> releaseBuilder.obbMainHash = content
"obbPatchFile" -> releaseBuilder.obbPatch = content
"obbPatchFileSha256" -> releaseBuilder.obbPatchHash = content
"permissions" -> releaseBuilder.permissions += content.split(',').filter { it.isNotEmpty() }
"features" -> releaseBuilder.features += content.split(',').filter { it.isNotEmpty() }
"nativecode" -> releaseBuilder.platforms += content.split(',').filter { it.isNotEmpty() }
}
}
productBuilder != null -> {
when (localName) {
"name" -> productBuilder.name = content
"summary" -> productBuilder.summary = content
"description" -> productBuilder.description = "<p>$content</p>"
"desc" -> productBuilder.description = content.replace("\n", "<br/>")
"icon" -> productBuilder.icon = validateIcon(content)
"author" -> productBuilder.authorName = content
"email" -> productBuilder.authorEmail = content
"source" -> productBuilder.source = content
"changelog" -> productBuilder.changelog = content
"web" -> productBuilder.web = content
"tracker" -> productBuilder.tracker = content
"added" -> productBuilder.added = content.parseDate()
"lastupdated" -> productBuilder.updated = content.parseDate()
"marketvercode" -> productBuilder.suggestedVersionCode = content.toLongOrNull() ?: 0L
"categories" -> productBuilder.categories += content.split(',').filter { it.isNotEmpty() }
"antifeatures" -> productBuilder.antiFeatures += content.split(',').filter { it.isNotEmpty() }
"license" -> productBuilder.licenses += content.split(',').filter { it.isNotEmpty() }
"donate" -> productBuilder.donates += Product.Donate.Regular(content)
"bitcoin" -> productBuilder.donates += Product.Donate.Bitcoin(content)
"litecoin" -> productBuilder.donates += Product.Donate.Litecoin(content)
"flattr" -> productBuilder.donates += Product.Donate.Flattr(content)
"liberapay" -> productBuilder.donates += Product.Donate.Liberapay(content)
"openCollective" -> productBuilder.donates += Product.Donate.OpenCollective(content)
}
}
fun onProduct(product: Product)
}
}
override fun characters(ch: CharArray, start: Int, length: Int) {
super.characters(ch, start, length)
contentBuilder.append(ch, start, length)
}
internal object DonateComparator : Comparator<Product.Donate> {
private val classes = listOf(
Product.Donate.Regular::class,
Product.Donate.Bitcoin::class,
Product.Donate.Litecoin::class,
Product.Donate.Flattr::class,
Product.Donate.Liberapay::class,
Product.Donate.OpenCollective::class
)
override fun compare(donate1: Product.Donate, donate2: Product.Donate): Int {
val index1 = classes.indexOf(donate1::class)
val index2 = classes.indexOf(donate2::class)
return when {
index1 >= 0 && index2 == -1 -> -1
index2 >= 0 && index1 == -1 -> 1
else -> index1.compareTo(index2)
}
}
}
private class RepositoryBuilder {
var address = ""
val mirrors = mutableListOf<String>()
var name = ""
var description = ""
var certificate = ""
var version = -1
var timestamp = 0L
}
private class ProductBuilder(val repositoryId: Long, val packageName: String) {
var name = ""
var summary = ""
var description = ""
var icon = ""
var authorName = ""
var authorEmail = ""
var source = ""
var changelog = ""
var web = ""
var tracker = ""
var added = 0L
var updated = 0L
var suggestedVersionCode = 0L
val categories = linkedSetOf<String>()
val antiFeatures = linkedSetOf<String>()
val licenses = mutableListOf<String>()
val donates = mutableListOf<Product.Donate>()
val releases = mutableListOf<Release>()
fun build(): Product {
return Product(
repositoryId,
packageName,
name,
summary,
description,
"",
icon,
"",
Product.Author(authorName, authorEmail, ""),
source,
changelog,
web,
tracker,
added,
updated,
suggestedVersionCode,
categories.toList(),
antiFeatures.toList(),
licenses,
donates.sortedWith(DonateComparator),
emptyList(),
releases
)
}
}
private class ReleaseBuilder {
var version = ""
var versionCode = 0L
var added = 0L
var size = 0L
var minSdkVersion = 0
var targetSdkVersion = 0
var maxSdkVersion = 0
var source = ""
var release = ""
var hash = ""
var hashType = ""
var signature = ""
var obbMain = ""
var obbMainHash = ""
var obbPatch = ""
var obbPatchHash = ""
val permissions = linkedSetOf<String>()
val features = linkedSetOf<String>()
val platforms = linkedSetOf<String>()
fun build(): Release {
val hashType = if (hash.isNotEmpty() && hashType.isEmpty()) "sha256" else hashType
val obbMainHashType = if (obbMainHash.isNotEmpty()) "sha256" else ""
val obbPatchHashType = if (obbPatchHash.isNotEmpty()) "sha256" else ""
return Release(
false,
version,
versionCode,
added,
size,
minSdkVersion,
targetSdkVersion,
maxSdkVersion,
source,
release,
hash,
hashType,
signature,
obbMain,
obbMainHash,
obbMainHashType,
obbPatch,
obbPatchHash,
obbPatchHashType,
permissions.toList(),
features.toList(),
platforms.toList(),
emptyList()
)
}
}
private val contentBuilder = StringBuilder()
private var repositoryBuilder: RepositoryBuilder? = RepositoryBuilder()
private var productBuilder: ProductBuilder? = null
private var releaseBuilder: ReleaseBuilder? = null
private fun Attributes.get(localName: String): String = getValue("", localName).orEmpty()
private fun String.cleanWhiteSpace(): String = replace("\\s".toRegex(), " ")
override fun startElement(
uri: String,
localName: String,
qName: String,
attributes: Attributes
) {
super.startElement(uri, localName, qName, attributes)
val repositoryBuilder = repositoryBuilder
val productBuilder = productBuilder
val releaseBuilder = releaseBuilder
contentBuilder.setLength(0)
when {
localName == "repo" -> {
if (repositoryBuilder != null) {
repositoryBuilder.address = attributes.get("url").cleanWhiteSpace()
repositoryBuilder.name = attributes.get("name").cleanWhiteSpace()
repositoryBuilder.description = attributes.get("description").cleanWhiteSpace()
repositoryBuilder.certificate = attributes.get("pubkey")
repositoryBuilder.version = attributes.get("version").toIntOrNull() ?: 0
repositoryBuilder.timestamp =
(attributes.get("timestamp").toLongOrNull() ?: 0L) * 1000L
}
}
localName == "application" && productBuilder == null -> {
this.productBuilder = ProductBuilder(repositoryId, attributes.get("id"))
}
localName == "package" && productBuilder != null && releaseBuilder == null -> {
this.releaseBuilder = ReleaseBuilder()
}
localName == "hash" && releaseBuilder != null -> {
releaseBuilder.hashType = attributes.get("type")
}
(localName == "uses-permission" || localName.startsWith("uses-permission-")) && releaseBuilder != null -> {
val minSdkVersion = if (localName != "uses-permission") {
"uses-permission-sdk-(\\d+)".toRegex().matchEntire(localName)
?.destructured?.let { (version) -> version.toIntOrNull() }
} else {
null
} ?: 0
val maxSdkVersion = attributes.get("maxSdkVersion").toIntOrNull() ?: Int.MAX_VALUE
if (Android.sdk in minSdkVersion..maxSdkVersion) {
releaseBuilder.permissions.add(attributes.get("name"))
} else {
releaseBuilder.permissions.remove(attributes.get("name"))
}
}
}
}
override fun endElement(uri: String, localName: String, qName: String) {
super.endElement(uri, localName, qName)
val repositoryBuilder = repositoryBuilder
val productBuilder = productBuilder
val releaseBuilder = releaseBuilder
val content = contentBuilder.toString()
when {
localName == "repo" -> {
if (repositoryBuilder != null) {
val mirrors = (listOf(repositoryBuilder.address) + repositoryBuilder.mirrors)
.filter { it.isNotEmpty() }.distinct()
callback.onRepository(
mirrors,
repositoryBuilder.name,
repositoryBuilder.description,
repositoryBuilder.certificate,
repositoryBuilder.version,
repositoryBuilder.timestamp
)
this.repositoryBuilder = null
}
}
localName == "application" && productBuilder != null -> {
val product = productBuilder.build()
this.productBuilder = null
callback.onProduct(product)
}
localName == "package" && productBuilder != null && releaseBuilder != null -> {
productBuilder.releases.add(releaseBuilder.build())
this.releaseBuilder = null
}
repositoryBuilder != null -> {
when (localName) {
"description" -> repositoryBuilder.description = content.cleanWhiteSpace()
"mirror" -> repositoryBuilder.mirrors += content
}
}
productBuilder != null && releaseBuilder != null -> {
when (localName) {
"version" -> releaseBuilder.version = content
"versioncode" -> releaseBuilder.versionCode = content.toLongOrNull() ?: 0L
"added" -> releaseBuilder.added = content.parseDate()
"size" -> releaseBuilder.size = content.toLongOrNull() ?: 0
"sdkver" -> releaseBuilder.minSdkVersion = content.toIntOrNull() ?: 0
"targetSdkVersion" -> releaseBuilder.targetSdkVersion =
content.toIntOrNull() ?: 0
"maxsdkver" -> releaseBuilder.maxSdkVersion = content.toIntOrNull() ?: 0
"srcname" -> releaseBuilder.source = content
"apkname" -> releaseBuilder.release = content
"hash" -> releaseBuilder.hash = content
"sig" -> releaseBuilder.signature = content
"obbMainFile" -> releaseBuilder.obbMain = content
"obbMainFileSha256" -> releaseBuilder.obbMainHash = content
"obbPatchFile" -> releaseBuilder.obbPatch = content
"obbPatchFileSha256" -> releaseBuilder.obbPatchHash = content
"permissions" -> releaseBuilder.permissions += content.split(',')
.filter { it.isNotEmpty() }
"features" -> releaseBuilder.features += content.split(',')
.filter { it.isNotEmpty() }
"nativecode" -> releaseBuilder.platforms += content.split(',')
.filter { it.isNotEmpty() }
}
}
productBuilder != null -> {
when (localName) {
"name" -> productBuilder.name = content
"summary" -> productBuilder.summary = content
"description" -> productBuilder.description = "<p>$content</p>"
"desc" -> productBuilder.description = content.replace("\n", "<br/>")
"icon" -> productBuilder.icon = validateIcon(content)
"author" -> productBuilder.authorName = content
"email" -> productBuilder.authorEmail = content
"source" -> productBuilder.source = content
"changelog" -> productBuilder.changelog = content
"web" -> productBuilder.web = content
"tracker" -> productBuilder.tracker = content
"added" -> productBuilder.added = content.parseDate()
"lastupdated" -> productBuilder.updated = content.parseDate()
"marketvercode" -> productBuilder.suggestedVersionCode =
content.toLongOrNull() ?: 0L
"categories" -> productBuilder.categories += content.split(',')
.filter { it.isNotEmpty() }
"antifeatures" -> productBuilder.antiFeatures += content.split(',')
.filter { it.isNotEmpty() }
"license" -> productBuilder.licenses += content.split(',')
.filter { it.isNotEmpty() }
"donate" -> productBuilder.donates += Product.Donate.Regular(content)
"bitcoin" -> productBuilder.donates += Product.Donate.Bitcoin(content)
"litecoin" -> productBuilder.donates += Product.Donate.Litecoin(content)
"flattr" -> productBuilder.donates += Product.Donate.Flattr(content)
"liberapay" -> productBuilder.donates += Product.Donate.Liberapay(content)
"openCollective" -> productBuilder.donates += Product.Donate.OpenCollective(
content
)
}
}
}
}
override fun characters(ch: CharArray, start: Int, length: Int) {
super.characters(ch, start, length)
contentBuilder.append(ch, start, length)
}
}

140
src/main/kotlin/com/looker/droidify/index/IndexMerger.kt
View File

@ -5,79 +5,93 @@ import android.database.sqlite.SQLiteDatabase
import com.fasterxml.jackson.core.JsonToken
import com.looker.droidify.entity.Product
import com.looker.droidify.entity.Release
import com.looker.droidify.utility.extension.android.*
import com.looker.droidify.utility.extension.json.*
import com.looker.droidify.utility.extension.android.asSequence
import com.looker.droidify.utility.extension.android.execWithResult
import com.looker.droidify.utility.extension.json.Json
import com.looker.droidify.utility.extension.json.collectNotNull
import com.looker.droidify.utility.extension.json.writeDictionary
import java.io.ByteArrayOutputStream
import java.io.Closeable
import java.io.File
class IndexMerger(file: File): Closeable {
private val db = SQLiteDatabase.openOrCreateDatabase(file, null)
class IndexMerger(file: File) : Closeable {
private val db = SQLiteDatabase.openOrCreateDatabase(file, null)
init {
db.execWithResult("PRAGMA synchronous = OFF")
db.execWithResult("PRAGMA journal_mode = OFF")
db.execSQL("CREATE TABLE product (package_name TEXT PRIMARY KEY, description TEXT NOT NULL, data BLOB NOT NULL)")
db.execSQL("CREATE TABLE releases (package_name TEXT PRIMARY KEY, data BLOB NOT NULL)")
db.beginTransaction()
}
fun addProducts(products: List<Product>) {
for (product in products) {
val outputStream = ByteArrayOutputStream()
Json.factory.createGenerator(outputStream).use { it.writeDictionary(product::serialize) }
db.insert("product", null, ContentValues().apply {
put("package_name", product.packageName)
put("description", product.description)
put("data", outputStream.toByteArray())
})
init {
db.execWithResult("PRAGMA synchronous = OFF")
db.execWithResult("PRAGMA journal_mode = OFF")
db.execSQL("CREATE TABLE product (package_name TEXT PRIMARY KEY, description TEXT NOT NULL, data BLOB NOT NULL)")
db.execSQL("CREATE TABLE releases (package_name TEXT PRIMARY KEY, data BLOB NOT NULL)")
db.beginTransaction()
}
}
fun addReleases(pairs: List<Pair<String, List<Release>>>) {
for (pair in pairs) {
val (packageName, releases) = pair
val outputStream = ByteArrayOutputStream()
Json.factory.createGenerator(outputStream).use {
it.writeStartArray()
for (release in releases) {
it.writeDictionary(release::serialize)
fun addProducts(products: List<Product>) {
for (product in products) {
val outputStream = ByteArrayOutputStream()
Json.factory.createGenerator(outputStream)
.use { it.writeDictionary(product::serialize) }
db.insert("product", null, ContentValues().apply {
put("package_name", product.packageName)
put("description", product.description)
put("data", outputStream.toByteArray())
})
}
it.writeEndArray()
}
db.insert("releases", null, ContentValues().apply {
put("package_name", packageName)
put("data", outputStream.toByteArray())
})
}
}
private fun closeTransaction() {
if (db.inTransaction()) {
db.setTransactionSuccessful()
db.endTransaction()
}
}
fun forEach(repositoryId: Long, windowSize: Int, callback: (List<Product>, Int) -> Unit) {
closeTransaction()
db.rawQuery("""SELECT product.description, product.data AS pd, releases.data AS rd FROM product
LEFT JOIN releases ON product.package_name = releases.package_name""", null)
?.use { it.asSequence().map {
val description = it.getString(0)
val product = Json.factory.createParser(it.getBlob(1)).use {
it.nextToken()
Product.deserialize(repositoryId, description, it)
fun addReleases(pairs: List<Pair<String, List<Release>>>) {
for (pair in pairs) {
val (packageName, releases) = pair
val outputStream = ByteArrayOutputStream()
Json.factory.createGenerator(outputStream).use {
it.writeStartArray()
for (release in releases) {
it.writeDictionary(release::serialize)
}
it.writeEndArray()
}
db.insert("releases", null, ContentValues().apply {
put("package_name", packageName)
put("data", outputStream.toByteArray())
})
}
val releases = it.getBlob(2)?.let { Json.factory.createParser(it).use {
it.nextToken()
it.collectNotNull(JsonToken.START_OBJECT, Release.Companion::deserialize)
} }.orEmpty()
product.copy(releases = releases)
}.windowed(windowSize, windowSize, true).forEach { products -> callback(products, it.count) } }
}
}
override fun close() {
db.use { closeTransaction() }
}
private fun closeTransaction() {
if (db.inTransaction()) {
db.setTransactionSuccessful()
db.endTransaction()
}
}
fun forEach(repositoryId: Long, windowSize: Int, callback: (List<Product>, Int) -> Unit) {
closeTransaction()
db.rawQuery(
"""SELECT product.description, product.data AS pd, releases.data AS rd FROM product
LEFT JOIN releases ON product.package_name = releases.package_name""", null
)
?.use { it ->
it.asSequence().map {
val description = it.getString(0)
val product = Json.factory.createParser(it.getBlob(1)).use {
it.nextToken()
Product.deserialize(repositoryId, description, it)
}
val releases = it.getBlob(2)?.let {
Json.factory.createParser(it).use {
it.nextToken()
it.collectNotNull(
JsonToken.START_OBJECT,
Release.Companion::deserialize
)
}
}.orEmpty()
product.copy(releases = releases)
}.windowed(windowSize, windowSize, true)
.forEach { products -> callback(products, it.count) }
}
}
override fun close() {
db.use { closeTransaction() }
}
}

554
src/main/kotlin/com/looker/droidify/index/IndexV1Parser.kt
View File

@ -4,257 +4,349 @@ import com.fasterxml.jackson.core.JsonParser
import com.fasterxml.jackson.core.JsonToken
import com.looker.droidify.entity.Product
import com.looker.droidify.entity.Release
import com.looker.droidify.utility.extension.android.*
import com.looker.droidify.utility.extension.android.Android
import com.looker.droidify.utility.extension.json.*
import com.looker.droidify.utility.extension.text.*
import com.looker.droidify.utility.extension.text.nullIfEmpty
import java.io.InputStream
object IndexV1Parser {
interface Callback {
fun onRepository(mirrors: List<String>, name: String, description: String, version: Int, timestamp: Long)
fun onProduct(product: Product)
fun onReleases(packageName: String, releases: List<Release>)
}
interface Callback {
fun onRepository(
mirrors: List<String>,
name: String,
description: String,
version: Int,
timestamp: Long
)
private class Screenshots(val phone: List<String>, val smallTablet: List<String>, val largeTablet: List<String>)
private class Localized(val name: String, val summary: String, val description: String,
val whatsNew: String, val metadataIcon: String, val screenshots: Screenshots?)
fun onProduct(product: Product)
fun onReleases(packageName: String, releases: List<Release>)
}
private fun <T> Map<String, Localized>.getAndCall(key: String, callback: (String, Localized) -> T?): T? {
return this[key]?.let { callback(key, it) }
}
private class Screenshots(
val phone: List<String>,
val smallTablet: List<String>,
val largeTablet: List<String>
)
private fun <T> Map<String, Localized>.find(callback: (String, Localized) -> T?): T? {
return getAndCall("en-US", callback) ?: getAndCall("en_US", callback) ?: getAndCall("en", callback)
}
private class Localized(
val name: String, val summary: String, val description: String,
val whatsNew: String, val metadataIcon: String, val screenshots: Screenshots?
)
private fun Map<String, Localized>.findString(fallback: String, callback: (Localized) -> String): String {
return (find { _, localized -> callback(localized).nullIfEmpty() } ?: fallback).trim()
}
private fun <T> Map<String, Localized>.getAndCall(
key: String,
callback: (String, Localized) -> T?
): T? {
return this[key]?.let { callback(key, it) }
}
fun parse(repositoryId: Long, inputStream: InputStream, callback: Callback) {
val jsonParser = Json.factory.createParser(inputStream)
if (jsonParser.nextToken() != JsonToken.START_OBJECT) {
jsonParser.illegal()
} else {
jsonParser.forEachKey {
when {
it.dictionary("repo") -> {
var address = ""
var mirrors = emptyList<String>()
var name = ""
var description = ""
var version = 0
var timestamp = 0L
forEachKey {
when {
it.string("address") -> address = valueAsString
it.array("mirrors") -> mirrors = collectDistinctNotEmptyStrings()
it.string("name") -> name = valueAsString
it.string("description") -> description = valueAsString
it.number("version") -> version = valueAsInt
it.number("timestamp") -> timestamp = valueAsLong
private fun <T> Map<String, Localized>.find(callback: (String, Localized) -> T?): T? {
return getAndCall("en-US", callback) ?: getAndCall("en_US", callback) ?: getAndCall(
"en",
callback
)
}
private fun Map<String, Localized>.findString(
fallback: String,
callback: (Localized) -> String
): String {
return (find { _, localized -> callback(localized).nullIfEmpty() } ?: fallback).trim()
}
fun parse(repositoryId: Long, inputStream: InputStream, callback: Callback) {
val jsonParser = Json.factory.createParser(inputStream)
if (jsonParser.nextToken() != JsonToken.START_OBJECT) {
jsonParser.illegal()
} else {
jsonParser.forEachKey { it ->
when {
it.dictionary("repo") -> {
var address = ""
var mirrors = emptyList<String>()
var name = ""
var description = ""
var version = 0
var timestamp = 0L
forEachKey {
when {
it.string("address") -> address = valueAsString
it.array("mirrors") -> mirrors = collectDistinctNotEmptyStrings()
it.string("name") -> name = valueAsString
it.string("description") -> description = valueAsString
it.number("version") -> version = valueAsInt
it.number("timestamp") -> timestamp = valueAsLong
else -> skipChildren()
}
}
val realMirrors =
((if (address.isNotEmpty()) listOf(address) else emptyList()) + mirrors).distinct()
callback.onRepository(realMirrors, name, description, version, timestamp)
}
it.array("apps") -> forEach(JsonToken.START_OBJECT) {
val product = parseProduct(repositoryId)
callback.onProduct(product)
}
it.dictionary("packages") -> forEachKey {
if (it.token == JsonToken.START_ARRAY) {
val packageName = it.key
val releases = collectNotNull(JsonToken.START_OBJECT) { parseRelease() }
callback.onReleases(packageName, releases)
} else {
skipChildren()
}
}
else -> skipChildren()
}
}
}
}
private fun JsonParser.parseProduct(repositoryId: Long): Product {
var packageName = ""
var nameFallback = ""
var summaryFallback = ""
var descriptionFallback = ""
var icon = ""
var authorName = ""
var authorEmail = ""
var authorWeb = ""
var source = ""
var changelog = ""
var web = ""
var tracker = ""
var added = 0L
var updated = 0L
var suggestedVersionCode = 0L
var categories = emptyList<String>()
var antiFeatures = emptyList<String>()
val licenses = mutableListOf<String>()
val donates = mutableListOf<Product.Donate>()
val localizedMap = mutableMapOf<String, Localized>()
forEachKey { it ->
when {
it.string("packageName") -> packageName = valueAsString
it.string("name") -> nameFallback = valueAsString
it.string("summary") -> summaryFallback = valueAsString
it.string("description") -> descriptionFallback = valueAsString
it.string("icon") -> icon = IndexHandler.validateIcon(valueAsString)
it.string("authorName") -> authorName = valueAsString
it.string("authorEmail") -> authorEmail = valueAsString
it.string("authorWebSite") -> authorWeb = valueAsString
it.string("sourceCode") -> source = valueAsString
it.string("changelog") -> changelog = valueAsString
it.string("webSite") -> web = valueAsString
it.string("issueTracker") -> tracker = valueAsString
it.number("added") -> added = valueAsLong
it.number("lastUpdated") -> updated = valueAsLong
it.string("suggestedVersionCode") -> suggestedVersionCode =
valueAsString.toLongOrNull() ?: 0L
it.array("categories") -> categories = collectDistinctNotEmptyStrings()
it.array("antiFeatures") -> antiFeatures = collectDistinctNotEmptyStrings()
it.string("license") -> licenses += valueAsString.split(',')
.filter { it.isNotEmpty() }
it.string("donate") -> donates += Product.Donate.Regular(valueAsString)
it.string("bitcoin") -> donates += Product.Donate.Bitcoin(valueAsString)
it.string("flattrID") -> donates += Product.Donate.Flattr(valueAsString)
it.string("liberapayID") -> donates += Product.Donate.Liberapay(valueAsString)
it.string("openCollective") -> donates += Product.Donate.OpenCollective(
valueAsString
)
it.dictionary("localized") -> forEachKey { it ->
if (it.token == JsonToken.START_OBJECT) {
val locale = it.key
var name = ""
var summary = ""
var description = ""
var whatsNew = ""
var metadataIcon = ""
var phone = emptyList<String>()
var smallTablet = emptyList<String>()
var largeTablet = emptyList<String>()
forEachKey {
when {
it.string("name") -> name = valueAsString
it.string("summary") -> summary = valueAsString
it.string("description") -> description = valueAsString
it.string("whatsNew") -> whatsNew = valueAsString
it.string("icon") -> metadataIcon = valueAsString
it.array("phoneScreenshots") -> phone =
collectDistinctNotEmptyStrings()
it.array("sevenInchScreenshots") -> smallTablet =
collectDistinctNotEmptyStrings()
it.array("tenInchScreenshots") -> largeTablet =
collectDistinctNotEmptyStrings()
else -> skipChildren()
}
}
val screenshots =
if (sequenceOf(phone, smallTablet, largeTablet).any { it.isNotEmpty() })
Screenshots(phone, smallTablet, largeTablet) else null
localizedMap[locale] = Localized(
name, summary, description, whatsNew,
metadataIcon.nullIfEmpty()?.let { "$locale/$it" }.orEmpty(), screenshots
)
} else {
skipChildren()
}
}
else -> skipChildren()
}
}
val realMirrors = ((if (address.isNotEmpty()) listOf(address) else emptyList()) + mirrors).distinct()
callback.onRepository(realMirrors, name, description, version, timestamp)
}
it.array("apps") -> forEach(JsonToken.START_OBJECT) {
val product = parseProduct(repositoryId)
callback.onProduct(product)
}
it.dictionary("packages") -> forEachKey {
if (it.token == JsonToken.START_ARRAY) {
val packageName = it.key
val releases = collectNotNull(JsonToken.START_OBJECT) { parseRelease() }
callback.onReleases(packageName, releases)
} else {
skipChildren()
}
}
else -> skipChildren()
}
}
val name = localizedMap.findString(nameFallback) { it.name }
val summary = localizedMap.findString(summaryFallback) { it.summary }
val description =
localizedMap.findString(descriptionFallback) { it.description }.replace("\n", "<br/>")
val whatsNew = localizedMap.findString("") { it.whatsNew }.replace("\n", "<br/>")
val metadataIcon = localizedMap.findString("") { it.metadataIcon }
val screenshotPairs =
localizedMap.find { key, localized -> localized.screenshots?.let { Pair(key, it) } }
val screenshots = screenshotPairs
?.let { (key, screenshots) ->
screenshots.phone.asSequence()
.map { Product.Screenshot(key, Product.Screenshot.Type.PHONE, it) } +
screenshots.smallTablet.asSequence()
.map {
Product.Screenshot(
key,
Product.Screenshot.Type.SMALL_TABLET,
it
)
} +
screenshots.largeTablet.asSequence()
.map {
Product.Screenshot(
key,
Product.Screenshot.Type.LARGE_TABLET,
it
)
}
}
.orEmpty().toList()
return Product(
repositoryId,
packageName,
name,
summary,
description,
whatsNew,
icon,
metadataIcon,
Product.Author(authorName, authorEmail, authorWeb),
source,
changelog,
web,
tracker,
added,
updated,
suggestedVersionCode,
categories,
antiFeatures,
licenses,
donates.sortedWith(IndexHandler.DonateComparator),
screenshots,
emptyList()
)
}
}
private fun JsonParser.parseProduct(repositoryId: Long): Product {
var packageName = ""
var nameFallback = ""
var summaryFallback = ""
var descriptionFallback = ""
var icon = ""
var authorName = ""
var authorEmail = ""
var authorWeb = ""
var source = ""
var changelog = ""
var web = ""
var tracker = ""
var added = 0L
var updated = 0L
var suggestedVersionCode = 0L
var categories = emptyList<String>()
var antiFeatures = emptyList<String>()
val licenses = mutableListOf<String>()
val donates = mutableListOf<Product.Donate>()
val localizedMap = mutableMapOf<String, Localized>()
forEachKey {
when {
it.string("packageName") -> packageName = valueAsString
it.string("name") -> nameFallback = valueAsString
it.string("summary") -> summaryFallback = valueAsString
it.string("description") -> descriptionFallback = valueAsString
it.string("icon") -> icon = IndexHandler.validateIcon(valueAsString)
it.string("authorName") -> authorName = valueAsString
it.string("authorEmail") -> authorEmail = valueAsString
it.string("authorWebSite") -> authorWeb = valueAsString
it.string("sourceCode") -> source = valueAsString
it.string("changelog") -> changelog = valueAsString
it.string("webSite") -> web = valueAsString
it.string("issueTracker") -> tracker = valueAsString
it.number("added") -> added = valueAsLong
it.number("lastUpdated") -> updated = valueAsLong
it.string("suggestedVersionCode") -> suggestedVersionCode = valueAsString.toLongOrNull() ?: 0L
it.array("categories") -> categories = collectDistinctNotEmptyStrings()
it.array("antiFeatures") -> antiFeatures = collectDistinctNotEmptyStrings()
it.string("license") -> licenses += valueAsString.split(',').filter { it.isNotEmpty() }
it.string("donate") -> donates += Product.Donate.Regular(valueAsString)
it.string("bitcoin") -> donates += Product.Donate.Bitcoin(valueAsString)
it.string("flattrID") -> donates += Product.Donate.Flattr(valueAsString)
it.string("liberapayID") -> donates += Product.Donate.Liberapay(valueAsString)
it.string("openCollective") -> donates += Product.Donate.OpenCollective(valueAsString)
it.dictionary("localized") -> forEachKey {
if (it.token == JsonToken.START_OBJECT) {
val locale = it.key
var name = ""
var summary = ""
var description = ""
var whatsNew = ""
var metadataIcon = ""
var phone = emptyList<String>()
var smallTablet = emptyList<String>()
var largeTablet = emptyList<String>()
forEachKey {
when {
it.string("name") -> name = valueAsString
it.string("summary") -> summary = valueAsString
it.string("description") -> description = valueAsString
it.string("whatsNew") -> whatsNew = valueAsString
it.string("icon") -> metadataIcon = valueAsString
it.array("phoneScreenshots") -> phone = collectDistinctNotEmptyStrings()
it.array("sevenInchScreenshots") -> smallTablet = collectDistinctNotEmptyStrings()
it.array("tenInchScreenshots") -> largeTablet = collectDistinctNotEmptyStrings()
private fun JsonParser.parseRelease(): Release {
var version = ""
var versionCode = 0L
var added = 0L
var size = 0L
var minSdkVersion = 0
var targetSdkVersion = 0
var maxSdkVersion = 0
var source = ""
var release = ""
var hash = ""
var hashTypeCandidate = ""
var signature = ""
var obbMain = ""
var obbMainHash = ""
var obbPatch = ""
var obbPatchHash = ""
val permissions = linkedSetOf<String>()
var features = emptyList<String>()
var platforms = emptyList<String>()
forEachKey {
when {
it.string("versionName") -> version = valueAsString
it.number("versionCode") -> versionCode = valueAsLong
it.number("added") -> added = valueAsLong
it.number("size") -> size = valueAsLong
it.number("minSdkVersion") -> minSdkVersion = valueAsInt
it.number("targetSdkVersion") -> targetSdkVersion = valueAsInt
it.number("maxSdkVersion") -> maxSdkVersion = valueAsInt
it.string("srcname") -> source = valueAsString
it.string("apkName") -> release = valueAsString
it.string("hash") -> hash = valueAsString
it.string("hashType") -> hashTypeCandidate = valueAsString
it.string("sig") -> signature = valueAsString
it.string("obbMainFile") -> obbMain = valueAsString
it.string("obbMainFileSha256") -> obbMainHash = valueAsString
it.string("obbPatchFile") -> obbPatch = valueAsString
it.string("obbPatchFileSha256") -> obbPatchHash = valueAsString
it.array("uses-permission") -> collectPermissions(permissions, 0)
it.array("uses-permission-sdk-23") -> collectPermissions(permissions, 23)
it.array("features") -> features = collectDistinctNotEmptyStrings()
it.array("nativecode") -> platforms = collectDistinctNotEmptyStrings()
else -> skipChildren()
}
}
val screenshots = if (sequenceOf(phone, smallTablet, largeTablet).any { it.isNotEmpty() })
Screenshots(phone, smallTablet, largeTablet) else null
localizedMap[locale] = Localized(name, summary, description, whatsNew,
metadataIcon.nullIfEmpty()?.let { "$locale/$it" }.orEmpty(), screenshots)
} else {
skipChildren()
}
}
else -> skipChildren()
}
val hashType =
if (hash.isNotEmpty() && hashTypeCandidate.isEmpty()) "sha256" else hashTypeCandidate
val obbMainHashType = if (obbMainHash.isNotEmpty()) "sha256" else ""
val obbPatchHashType = if (obbPatchHash.isNotEmpty()) "sha256" else ""
return Release(
false,
version,
versionCode,
added,
size,
minSdkVersion,
targetSdkVersion,
maxSdkVersion,
source,
release,
hash,
hashType,
signature,
obbMain,
obbMainHash,
obbMainHashType,
obbPatch,
obbPatchHash,
obbPatchHashType,
permissions.toList(),
features,
platforms,
emptyList()
)
}
val name = localizedMap.findString(nameFallback) { it.name }
val summary = localizedMap.findString(summaryFallback) { it.summary }
val description = localizedMap.findString(descriptionFallback) { it.description }.replace("\n", "<br/>")
val whatsNew = localizedMap.findString("") { it.whatsNew }.replace("\n", "<br/>")
val metadataIcon = localizedMap.findString("") { it.metadataIcon }
val screenshotPairs = localizedMap.find { key, localized -> localized.screenshots?.let { Pair(key, it) } }
val screenshots = screenshotPairs
?.let { (key, screenshots) -> screenshots.phone.asSequence()
.map { Product.Screenshot(key, Product.Screenshot.Type.PHONE, it) } +
screenshots.smallTablet.asSequence()
.map { Product.Screenshot(key, Product.Screenshot.Type.SMALL_TABLET, it) } +
screenshots.largeTablet.asSequence()
.map { Product.Screenshot(key, Product.Screenshot.Type.LARGE_TABLET, it) } }
.orEmpty().toList()
return Product(repositoryId, packageName, name, summary, description, whatsNew, icon, metadataIcon,
Product.Author(authorName, authorEmail, authorWeb), source, changelog, web, tracker, added, updated,
suggestedVersionCode, categories, antiFeatures, licenses,
donates.sortedWith(IndexHandler.DonateComparator), screenshots, emptyList())
}
private fun JsonParser.parseRelease(): Release {
var version = ""
var versionCode = 0L
var added = 0L
var size = 0L
var minSdkVersion = 0
var targetSdkVersion = 0
var maxSdkVersion = 0
var source = ""
var release = ""
var hash = ""
var hashTypeCandidate = ""
var signature = ""
var obbMain = ""
var obbMainHash = ""
var obbPatch = ""
var obbPatchHash = ""
val permissions = linkedSetOf<String>()
var features = emptyList<String>()
var platforms = emptyList<String>()
forEachKey {
when {
it.string("versionName") -> version = valueAsString
it.number("versionCode") -> versionCode = valueAsLong
it.number("added") -> added = valueAsLong
it.number("size") -> size = valueAsLong
it.number("minSdkVersion") -> minSdkVersion = valueAsInt
it.number("targetSdkVersion") -> targetSdkVersion = valueAsInt
it.number("maxSdkVersion") -> maxSdkVersion = valueAsInt
it.string("srcname") -> source = valueAsString
it.string("apkName") -> release = valueAsString
it.string("hash") -> hash = valueAsString
it.string("hashType") -> hashTypeCandidate = valueAsString
it.string("sig") -> signature = valueAsString
it.string("obbMainFile") -> obbMain = valueAsString
it.string("obbMainFileSha256") -> obbMainHash = valueAsString
it.string("obbPatchFile") -> obbPatch = valueAsString
it.string("obbPatchFileSha256") -> obbPatchHash = valueAsString
it.array("uses-permission") -> collectPermissions(permissions, 0)
it.array("uses-permission-sdk-23") -> collectPermissions(permissions, 23)
it.array("features") -> features = collectDistinctNotEmptyStrings()
it.array("nativecode") -> platforms = collectDistinctNotEmptyStrings()
else -> skipChildren()
}
}
val hashType = if (hash.isNotEmpty() && hashTypeCandidate.isEmpty()) "sha256" else hashTypeCandidate
val obbMainHashType = if (obbMainHash.isNotEmpty()) "sha256" else ""
val obbPatchHashType = if (obbPatchHash.isNotEmpty()) "sha256" else ""
return Release(false, version, versionCode, added, size,
minSdkVersion, targetSdkVersion, maxSdkVersion, source, release, hash, hashType, signature,
obbMain, obbMainHash, obbMainHashType, obbPatch, obbPatchHash, obbPatchHashType,
permissions.toList(), features, platforms, emptyList())
}
private fun JsonParser.collectPermissions(permissions: LinkedHashSet<String>, minSdk: Int) {
forEach(JsonToken.START_ARRAY) {
val firstToken = nextToken()
val permission = if (firstToken == JsonToken.VALUE_STRING) valueAsString else ""
if (firstToken != JsonToken.END_ARRAY) {
val secondToken = nextToken()
val maxSdk = if (secondToken == JsonToken.VALUE_NUMBER_INT) valueAsInt else 0
if (permission.isNotEmpty() && Android.sdk >= minSdk && (maxSdk <= 0 || Android.sdk <= maxSdk)) {
permissions.add(permission)
}
if (secondToken != JsonToken.END_ARRAY) {
while (true) {
val token = nextToken()
if (token == JsonToken.END_ARRAY) {
break
} else if (token.isStructStart) {
skipChildren()
private fun JsonParser.collectPermissions(permissions: LinkedHashSet<String>, minSdk: Int) {
forEach(JsonToken.START_ARRAY) {
val firstToken = nextToken()
val permission = if (firstToken == JsonToken.VALUE_STRING) valueAsString else ""
if (firstToken != JsonToken.END_ARRAY) {
val secondToken = nextToken()
val maxSdk = if (secondToken == JsonToken.VALUE_NUMBER_INT) valueAsInt else 0
if (permission.isNotEmpty() && Android.sdk >= minSdk && (maxSdk <= 0 || Android.sdk <= maxSdk)) {
permissions.add(permission)
}
if (secondToken != JsonToken.END_ARRAY) {
while (true) {
val token = nextToken()
if (token == JsonToken.END_ARRAY) {
break
} else if (token.isStructStart) {
skipChildren()
}
}
}
}
}
}
}
}
}
}

704
src/main/kotlin/com/looker/droidify/index/RepositoryUpdater.kt
View File

@ -2,9 +2,6 @@ package com.looker.droidify.index
import android.content.Context
import android.net.Uri
import io.reactivex.rxjava3.core.Observable
import io.reactivex.rxjava3.core.Single
import io.reactivex.rxjava3.schedulers.Schedulers
import com.looker.droidify.content.Cache
import com.looker.droidify.database.Database
import com.looker.droidify.entity.Product
@ -14,332 +11,455 @@ import com.looker.droidify.network.Downloader
import com.looker.droidify.utility.ProgressInputStream
import com.looker.droidify.utility.RxUtils
import com.looker.droidify.utility.Utils
import com.looker.droidify.utility.extension.android.*
import com.looker.droidify.utility.extension.text.*
import com.looker.droidify.utility.extension.android.Android
import com.looker.droidify.utility.extension.text.unhex
import io.reactivex.rxjava3.core.Observable
import io.reactivex.rxjava3.core.Single
import io.reactivex.rxjava3.schedulers.Schedulers
import org.xml.sax.InputSource
import java.io.File
import java.security.cert.X509Certificate
import java.util.Locale
import java.util.*
import java.util.jar.JarEntry
import java.util.jar.JarFile
import javax.xml.parsers.SAXParserFactory
object RepositoryUpdater {
enum class Stage {
DOWNLOAD, PROCESS, MERGE, COMMIT
}
private enum class IndexType(val jarName: String, val contentName: String, val certificateFromIndex: Boolean) {
INDEX("index.jar", "index.xml", true),
INDEX_V1("index-v1.jar", "index-v1.json", false)
}
enum class ErrorType {
NETWORK, HTTP, VALIDATION, PARSING
}
class UpdateException: Exception {
val errorType: ErrorType
constructor(errorType: ErrorType, message: String): super(message) {
this.errorType = errorType
enum class Stage {
DOWNLOAD, PROCESS, MERGE, COMMIT
}
constructor(errorType: ErrorType, message: String, cause: Exception): super(message, cause) {
this.errorType = errorType
private enum class IndexType(
val jarName: String,
val contentName: String,
val certificateFromIndex: Boolean
) {
INDEX("index.jar", "index.xml", true),
INDEX_V1("index-v1.jar", "index-v1.json", false)
}
}
private lateinit var context: Context
private val updaterLock = Any()
private val cleanupLock = Any()
enum class ErrorType {
NETWORK, HTTP, VALIDATION, PARSING
}
fun init(context: Context) {
this.context = context
class UpdateException : Exception {
val errorType: ErrorType
var lastDisabled = setOf<Long>()
Observable.just(Unit)
.concatWith(Database.observable(Database.Subject.Repositories))
.observeOn(Schedulers.io())
.flatMapSingle { RxUtils.querySingle { Database.RepositoryAdapter.getAllDisabledDeleted(it) } }
.forEach {
val newDisabled = it.asSequence().filter { !it.second }.map { it.first }.toSet()
val disabled = newDisabled - lastDisabled
lastDisabled = newDisabled
val deleted = it.asSequence().filter { it.second }.map { it.first }.toSet()
if (disabled.isNotEmpty() || deleted.isNotEmpty()) {
val pairs = (disabled.asSequence().map { Pair(it, false) } +
deleted.asSequence().map { Pair(it, true) }).toSet()
synchronized(cleanupLock) { Database.RepositoryAdapter.cleanup(pairs) }
constructor(errorType: ErrorType, message: String) : super(message) {
this.errorType = errorType
}
}
}
fun await() {
synchronized(updaterLock) { }
}
fun update(repository: Repository, unstable: Boolean,
callback: (Stage, Long, Long?) -> Unit): Single<Boolean> {
return update(repository, listOf(IndexType.INDEX_V1, IndexType.INDEX), unstable, callback)
}
private fun update(repository: Repository, indexTypes: List<IndexType>, unstable: Boolean,
callback: (Stage, Long, Long?) -> Unit): Single<Boolean> {
val indexType = indexTypes[0]
return downloadIndex(repository, indexType, callback)
.flatMap { (result, file) ->
when {
result.isNotChanged -> {
file.delete()
Single.just(false)
}
!result.success -> {
file.delete()
if (result.code == 404 && indexTypes.isNotEmpty()) {
update(repository, indexTypes.subList(1, indexTypes.size), unstable, callback)
} else {
Single.error(UpdateException(ErrorType.HTTP, "Invalid response: HTTP ${result.code}"))
}
}
else -> {
RxUtils.managedSingle { processFile(repository, indexType, unstable,
file, result.lastModified, result.entityTag, callback) }
}
constructor(errorType: ErrorType, message: String, cause: Exception) : super(
message,
cause
) {
this.errorType = errorType
}
}
}
}
private fun downloadIndex(repository: Repository, indexType: IndexType,
callback: (Stage, Long, Long?) -> Unit): Single<Pair<Downloader.Result, File>> {
return Single.just(Unit)
.map { Cache.getTemporaryFile(context) }
.flatMap { file -> Downloader
.download(Uri.parse(repository.address).buildUpon()
.appendPath(indexType.jarName).build().toString(), file, repository.lastModified, repository.entityTag,
repository.authentication) { read, total -> callback(Stage.DOWNLOAD, read, total) }
.subscribeOn(Schedulers.io())
.map { Pair(it, file) }
.onErrorResumeNext {
file.delete()
when (it) {
is InterruptedException, is RuntimeException, is Error -> Single.error(it)
is Exception -> Single.error(UpdateException(ErrorType.NETWORK, "Network error", it))
else -> Single.error(it)
}
} }
}
private lateinit var context: Context
private val updaterLock = Any()
private val cleanupLock = Any()
private fun processFile(repository: Repository, indexType: IndexType, unstable: Boolean,
file: File, lastModified: String, entityTag: String, callback: (Stage, Long, Long?) -> Unit): Boolean {
var rollback = true
return synchronized(updaterLock) {
try {
val jarFile = JarFile(file, true)
val indexEntry = jarFile.getEntry(indexType.contentName) as JarEntry
val total = indexEntry.size
Database.UpdaterAdapter.createTemporaryTable()
val features = context.packageManager.systemAvailableFeatures
.asSequence().map { it.name }.toSet() + setOf("android.hardware.touchscreen")
fun init(context: Context) {
this.context = context
val (changedRepository, certificateFromIndex) = when (indexType) {
IndexType.INDEX -> {
val factory = SAXParserFactory.newInstance()
factory.isNamespaceAware = true
val parser = factory.newSAXParser()
val reader = parser.xmlReader
var changedRepository: Repository? = null
var certificateFromIndex: String? = null
val products = mutableListOf<Product>()
reader.contentHandler = IndexHandler(repository.id, object: IndexHandler.Callback {
override fun onRepository(mirrors: List<String>, name: String, description: String,
certificate: String, version: Int, timestamp: Long) {
changedRepository = repository.update(mirrors, name, description, version,
lastModified, entityTag, timestamp)
certificateFromIndex = certificate.toLowerCase(Locale.US)
}
override fun onProduct(product: Product) {
if (Thread.interrupted()) {
throw InterruptedException()
var lastDisabled = setOf<Long>()
Observable.just(Unit)
.concatWith(Database.observable(Database.Subject.Repositories))
.observeOn(Schedulers.io())
.flatMapSingle {
RxUtils.querySingle {
Database.RepositoryAdapter.getAllDisabledDeleted(
it
)
}
products += transformProduct(product, features, unstable)
if (products.size >= 50) {
Database.UpdaterAdapter.putTemporary(products)
products.clear()
}
.forEach { it ->
val newDisabled = it.asSequence().filter { !it.second }.map { it.first }.toSet()
val disabled = newDisabled - lastDisabled
lastDisabled = newDisabled
val deleted = it.asSequence().filter { it.second }.map { it.first }.toSet()
if (disabled.isNotEmpty() || deleted.isNotEmpty()) {
val pairs = (disabled.asSequence().map { Pair(it, false) } +
deleted.asSequence().map { Pair(it, true) }).toSet()
synchronized(cleanupLock) { Database.RepositoryAdapter.cleanup(pairs) }
}
}
})
ProgressInputStream(jarFile.getInputStream(indexEntry)) { callback(Stage.PROCESS, it, total) }
.use { reader.parse(InputSource(it)) }
if (Thread.interrupted()) {
throw InterruptedException()
}
if (products.isNotEmpty()) {
Database.UpdaterAdapter.putTemporary(products)
products.clear()
}
Pair(changedRepository, certificateFromIndex)
}
IndexType.INDEX_V1 -> {
var changedRepository: Repository? = null
}
val mergerFile = Cache.getTemporaryFile(context)
fun await() {
synchronized(updaterLock) { }
}
fun update(
repository: Repository, unstable: Boolean,
callback: (Stage, Long, Long?) -> Unit
): Single<Boolean> {
return update(repository, listOf(IndexType.INDEX_V1, IndexType.INDEX), unstable, callback)
}
private fun update(
repository: Repository, indexTypes: List<IndexType>, unstable: Boolean,
callback: (Stage, Long, Long?) -> Unit
): Single<Boolean> {
val indexType = indexTypes[0]
return downloadIndex(repository, indexType, callback)
.flatMap { (result, file) ->
when {
result.isNotChanged -> {
file.delete()
Single.just(false)
}
!result.success -> {
file.delete()
if (result.code == 404 && indexTypes.isNotEmpty()) {
update(
repository,
indexTypes.subList(1, indexTypes.size),
unstable,
callback
)
} else {
Single.error(
UpdateException(
ErrorType.HTTP,
"Invalid response: HTTP ${result.code}"
)
)
}
}
else -> {
RxUtils.managedSingle {
processFile(
repository, indexType, unstable,
file, result.lastModified, result.entityTag, callback
)
}
}
}
}
}
private fun downloadIndex(
repository: Repository, indexType: IndexType,
callback: (Stage, Long, Long?) -> Unit
): Single<Pair<Downloader.Result, File>> {
return Single.just(Unit)
.map { Cache.getTemporaryFile(context) }
.flatMap { file ->
Downloader
.download(
Uri.parse(repository.address).buildUpon()
.appendPath(indexType.jarName).build().toString(),
file,
repository.lastModified,
repository.entityTag,
repository.authentication
) { read, total -> callback(Stage.DOWNLOAD, read, total) }
.subscribeOn(Schedulers.io())
.map { Pair(it, file) }
.onErrorResumeNext {
file.delete()
when (it) {
is InterruptedException, is RuntimeException, is Error -> Single.error(
it
)
is Exception -> Single.error(
UpdateException(
ErrorType.NETWORK,
"Network error",
it
)
)
else -> Single.error(it)
}
}
}
}
private fun processFile(
repository: Repository, indexType: IndexType, unstable: Boolean,
file: File, lastModified: String, entityTag: String, callback: (Stage, Long, Long?) -> Unit
): Boolean {
var rollback = true
return synchronized(updaterLock) {
try {
val unmergedProducts = mutableListOf<Product>()
val unmergedReleases = mutableListOf<Pair<String, List<Release>>>()
IndexMerger(mergerFile).use { indexMerger ->
ProgressInputStream(jarFile.getInputStream(indexEntry)) { callback(Stage.PROCESS, it, total) }.use {
IndexV1Parser.parse(repository.id, it, object: IndexV1Parser.Callback {
override fun onRepository(mirrors: List<String>, name: String, description: String,
version: Int, timestamp: Long) {
changedRepository = repository.update(mirrors, name, description, version,
lastModified, entityTag, timestamp)
}
val jarFile = JarFile(file, true)
val indexEntry = jarFile.getEntry(indexType.contentName) as JarEntry
val total = indexEntry.size
Database.UpdaterAdapter.createTemporaryTable()
val features = context.packageManager.systemAvailableFeatures
.asSequence().map { it.name }.toSet() + setOf("android.hardware.touchscreen")
override fun onProduct(product: Product) {
if (Thread.interrupted()) {
throw InterruptedException()
}
unmergedProducts += product
if (unmergedProducts.size >= 50) {
indexMerger.addProducts(unmergedProducts)
unmergedProducts.clear()
}
}
val (changedRepository, certificateFromIndex) = when (indexType) {
IndexType.INDEX -> {
val factory = SAXParserFactory.newInstance()
factory.isNamespaceAware = true
val parser = factory.newSAXParser()
val reader = parser.xmlReader
var changedRepository: Repository? = null
var certificateFromIndex: String? = null
val products = mutableListOf<Product>()
override fun onReleases(packageName: String, releases: List<Release>) {
if (Thread.interrupted()) {
throw InterruptedException()
}
unmergedReleases += Pair(packageName, releases)
if (unmergedReleases.size >= 50) {
indexMerger.addReleases(unmergedReleases)
unmergedReleases.clear()
}
}
})
reader.contentHandler =
IndexHandler(repository.id, object : IndexHandler.Callback {
override fun onRepository(
mirrors: List<String>, name: String, description: String,
certificate: String, version: Int, timestamp: Long
) {
changedRepository = repository.update(
mirrors, name, description, version,
lastModified, entityTag, timestamp
)
certificateFromIndex = certificate.lowercase(Locale.US)
}
if (Thread.interrupted()) {
throw InterruptedException()
}
if (unmergedProducts.isNotEmpty()) {
indexMerger.addProducts(unmergedProducts)
unmergedProducts.clear()
}
if (unmergedReleases.isNotEmpty()) {
indexMerger.addReleases(unmergedReleases)
unmergedReleases.clear()
}
var progress = 0
indexMerger.forEach(repository.id, 50) { products, totalCount ->
if (Thread.interrupted()) {
throw InterruptedException()
override fun onProduct(product: Product) {
if (Thread.interrupted()) {
throw InterruptedException()
}
products += transformProduct(product, features, unstable)
if (products.size >= 50) {
Database.UpdaterAdapter.putTemporary(products)
products.clear()
}
}
})
ProgressInputStream(jarFile.getInputStream(indexEntry)) {
callback(
Stage.PROCESS,
it,
total
)
}
.use { reader.parse(InputSource(it)) }
if (Thread.interrupted()) {
throw InterruptedException()
}
if (products.isNotEmpty()) {
Database.UpdaterAdapter.putTemporary(products)
products.clear()
}
Pair(changedRepository, certificateFromIndex)
}
IndexType.INDEX_V1 -> {
var changedRepository: Repository? = null
val mergerFile = Cache.getTemporaryFile(context)
try {
val unmergedProducts = mutableListOf<Product>()
val unmergedReleases = mutableListOf<Pair<String, List<Release>>>()
IndexMerger(mergerFile).use { indexMerger ->
ProgressInputStream(jarFile.getInputStream(indexEntry)) {
callback(
Stage.PROCESS,
it,
total
)
}.use { it ->
IndexV1Parser.parse(
repository.id,
it,
object : IndexV1Parser.Callback {
override fun onRepository(
mirrors: List<String>,
name: String,
description: String,
version: Int,
timestamp: Long
) {
changedRepository = repository.update(
mirrors, name, description, version,
lastModified, entityTag, timestamp
)
}
override fun onProduct(product: Product) {
if (Thread.interrupted()) {
throw InterruptedException()
}
unmergedProducts += product
if (unmergedProducts.size >= 50) {
indexMerger.addProducts(unmergedProducts)
unmergedProducts.clear()
}
}
override fun onReleases(
packageName: String,
releases: List<Release>
) {
if (Thread.interrupted()) {
throw InterruptedException()
}
unmergedReleases += Pair(packageName, releases)
if (unmergedReleases.size >= 50) {
indexMerger.addReleases(unmergedReleases)
unmergedReleases.clear()
}
}
})
if (Thread.interrupted()) {
throw InterruptedException()
}
if (unmergedProducts.isNotEmpty()) {
indexMerger.addProducts(unmergedProducts)
unmergedProducts.clear()
}
if (unmergedReleases.isNotEmpty()) {
indexMerger.addReleases(unmergedReleases)
unmergedReleases.clear()
}
var progress = 0
indexMerger.forEach(repository.id, 50) { products, totalCount ->
if (Thread.interrupted()) {
throw InterruptedException()
}
progress += products.size
callback(
Stage.MERGE,
progress.toLong(),
totalCount.toLong()
)
Database.UpdaterAdapter.putTemporary(products
.map { transformProduct(it, features, unstable) })
}
}
}
} finally {
mergerFile.delete()
}
Pair(changedRepository, null)
}
progress += products.size
callback(Stage.MERGE, progress.toLong(), totalCount.toLong())
Database.UpdaterAdapter.putTemporary(products
.map { transformProduct(it, features, unstable) })
}
}
}
} finally {
mergerFile.delete()
}
Pair(changedRepository, null)
}
}
val workRepository = changedRepository ?: repository
if (workRepository.timestamp < repository.timestamp) {
throw UpdateException(ErrorType.VALIDATION, "New index is older than current index: " +
"${workRepository.timestamp} < ${repository.timestamp}")
} else {
val fingerprint = run {
val certificateFromJar = run {
val codeSigners = indexEntry.codeSigners
if (codeSigners == null || codeSigners.size != 1) {
throw UpdateException(ErrorType.VALIDATION, "index.jar must be signed by a single code signer")
} else {
val certificates = codeSigners[0].signerCertPath?.certificates.orEmpty()
if (certificates.size != 1) {
throw UpdateException(ErrorType.VALIDATION, "index.jar code signer should have only one certificate")
val workRepository = changedRepository ?: repository
if (workRepository.timestamp < repository.timestamp) {
throw UpdateException(
ErrorType.VALIDATION, "New index is older than current index: " +
"${workRepository.timestamp} < ${repository.timestamp}"
)
} else {
certificates[0] as X509Certificate
val fingerprint = run {
val certificateFromJar = run {
val codeSigners = indexEntry.codeSigners
if (codeSigners == null || codeSigners.size != 1) {
throw UpdateException(
ErrorType.VALIDATION,
"index.jar must be signed by a single code signer"
)
} else {
val certificates =
codeSigners[0].signerCertPath?.certificates.orEmpty()
if (certificates.size != 1) {
throw UpdateException(
ErrorType.VALIDATION,
"index.jar code signer should have only one certificate"
)
} else {
certificates[0] as X509Certificate
}
}
}
val fingerprintFromJar = Utils.calculateFingerprint(certificateFromJar)
if (indexType.certificateFromIndex) {
val fingerprintFromIndex =
certificateFromIndex?.unhex()?.let(Utils::calculateFingerprint)
if (fingerprintFromIndex == null || fingerprintFromJar != fingerprintFromIndex) {
throw UpdateException(
ErrorType.VALIDATION,
"index.xml contains invalid public key"
)
}
fingerprintFromIndex
} else {
fingerprintFromJar
}
}
val commitRepository = if (workRepository.fingerprint != fingerprint) {
if (workRepository.fingerprint.isEmpty()) {
workRepository.copy(fingerprint = fingerprint)
} else {
throw UpdateException(
ErrorType.VALIDATION,
"Certificate fingerprints do not match"
)
}
} else {
workRepository
}
if (Thread.interrupted()) {
throw InterruptedException()
}
callback(Stage.COMMIT, 0, null)
synchronized(cleanupLock) {
Database.UpdaterAdapter.finishTemporary(
commitRepository,
true
)
}
rollback = false
true
}
} catch (e: Exception) {
throw when (e) {
is UpdateException, is InterruptedException -> e
else -> UpdateException(ErrorType.PARSING, "Error parsing index", e)
}
} finally {
file.delete()
if (rollback) {
Database.UpdaterAdapter.finishTemporary(repository, false)
}
}
}
val fingerprintFromJar = Utils.calculateFingerprint(certificateFromJar)
if (indexType.certificateFromIndex) {
val fingerprintFromIndex = certificateFromIndex?.unhex()?.let(Utils::calculateFingerprint)
if (fingerprintFromIndex == null || fingerprintFromJar != fingerprintFromIndex) {
throw UpdateException(ErrorType.VALIDATION, "index.xml contains invalid public key")
}
fingerprintFromIndex
} else {
fingerprintFromJar
}
}
val commitRepository = if (workRepository.fingerprint != fingerprint) {
if (workRepository.fingerprint.isEmpty()) {
workRepository.copy(fingerprint = fingerprint)
} else {
throw UpdateException(ErrorType.VALIDATION, "Certificate fingerprints do not match")
}
} else {
workRepository
}
if (Thread.interrupted()) {
throw InterruptedException()
}
callback(Stage.COMMIT, 0, null)
synchronized(cleanupLock) { Database.UpdaterAdapter.finishTemporary(commitRepository, true) }
rollback = false
true
}
} catch (e: Exception) {
throw when (e) {
is UpdateException, is InterruptedException -> e
else -> UpdateException(ErrorType.PARSING, "Error parsing index", e)
}
} finally {
file.delete()
if (rollback) {
Database.UpdaterAdapter.finishTemporary(repository, false)
}
}
}
}
private fun transformProduct(product: Product, features: Set<String>, unstable: Boolean): Product {
val releasePairs = product.releases.distinctBy { it.identifier }.sortedByDescending { it.versionCode }.map {
val incompatibilities = mutableListOf<Release.Incompatibility>()
if (it.minSdkVersion > 0 && Android.sdk < it.minSdkVersion) {
incompatibilities += Release.Incompatibility.MinSdk
}
if (it.maxSdkVersion > 0 && Android.sdk > it.maxSdkVersion) {
incompatibilities += Release.Incompatibility.MaxSdk
}
if (it.platforms.isNotEmpty() && it.platforms.intersect(Android.platforms).isEmpty()) {
incompatibilities += Release.Incompatibility.Platform
}
incompatibilities += (it.features - features).sorted().map { Release.Incompatibility.Feature(it) }
Pair(it, incompatibilities as List<Release.Incompatibility>)
}.toMutableList()
private fun transformProduct(
product: Product,
features: Set<String>,
unstable: Boolean
): Product {
val releasePairs =
product.releases.distinctBy { it.identifier }.sortedByDescending { it.versionCode }
.map { it ->
val incompatibilities = mutableListOf<Release.Incompatibility>()
if (it.minSdkVersion > 0 && Android.sdk < it.minSdkVersion) {
incompatibilities += Release.Incompatibility.MinSdk
}
if (it.maxSdkVersion > 0 && Android.sdk > it.maxSdkVersion) {
incompatibilities += Release.Incompatibility.MaxSdk
}
if (it.platforms.isNotEmpty() && it.platforms.intersect(Android.platforms)
.isEmpty()
) {
incompatibilities += Release.Incompatibility.Platform
}
incompatibilities += (it.features - features).sorted()
.map { Release.Incompatibility.Feature(it) }
Pair(it, incompatibilities as List<Release.Incompatibility>)
}.toMutableList()
val predicate: (Release) -> Boolean = { unstable || product.suggestedVersionCode <= 0 ||
it.versionCode <= product.suggestedVersionCode }
val firstCompatibleReleaseIndex = releasePairs.indexOfFirst { it.second.isEmpty() && predicate(it.first) }
val firstReleaseIndex = if (firstCompatibleReleaseIndex >= 0) firstCompatibleReleaseIndex else
releasePairs.indexOfFirst { predicate(it.first) }
val firstSelected = if (firstReleaseIndex >= 0) releasePairs[firstReleaseIndex] else null
val predicate: (Release) -> Boolean = {
unstable || product.suggestedVersionCode <= 0 ||
it.versionCode <= product.suggestedVersionCode
}
val firstCompatibleReleaseIndex =
releasePairs.indexOfFirst { it.second.isEmpty() && predicate(it.first) }
val firstReleaseIndex =
if (firstCompatibleReleaseIndex >= 0) firstCompatibleReleaseIndex else
releasePairs.indexOfFirst { predicate(it.first) }
val firstSelected = if (firstReleaseIndex >= 0) releasePairs[firstReleaseIndex] else null
val releases = releasePairs.map { (release, incompatibilities) -> release
.copy(incompatibilities = incompatibilities, selected = firstSelected
?.let { it.first.versionCode == release.versionCode && it.second == incompatibilities } == true) }
return product.copy(releases = releases)
}
val releases = releasePairs.map { (release, incompatibilities) ->
release
.copy(incompatibilities = incompatibilities, selected = firstSelected
?.let { it.first.versionCode == release.versionCode && it.second == incompatibilities } == true)
}
return product.copy(releases = releases)
}
}