From a94017d56418ba836becf72425ba067539479b5e Mon Sep 17 00:00:00 2001 From: kzaemrio Date: Thu, 2 Jun 2022 09:07:30 +0800 Subject: [PATCH] replace URL inputstream with okhttp inputstream in RssHelper (#85) * add okhttp3 in ext, and add okhttp-coroutines-jvm lib * replace URL inputstream with okhttp buffered inputstream in RssHelper * use executeAsync for parseFullContent in RssHelper * make searchFeed call without queryRssXml in RssHelper --- app/build.gradle | 3 +- .../reader/data/module/OkHttpClientModule.kt | 1 - .../ash/reader/data/repository/RssHelper.kt | 118 ++++++++++-------- 3 files changed, 65 insertions(+), 57 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index fbd5484..698cffa 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -113,10 +113,11 @@ dependencies { implementation("io.coil-kt:coil-svg:$coil") implementation("io.coil-kt:coil-gif:$coil") - implementation "org.conscrypt:conscrypt-android:2.5.2" + implementation 'org.conscrypt:conscrypt-android:2.5.2' // https://square.github.io/okhttp/changelogs/changelog/ implementation "com.squareup.okhttp3:okhttp:$okhttp" + implementation "com.squareup.okhttp3:okhttp-coroutines-jvm:$okhttp" implementation "com.squareup.retrofit2:retrofit:$retrofit2" implementation "com.squareup.retrofit2:converter-gson:$retrofit2" diff --git a/app/src/main/java/me/ash/reader/data/module/OkHttpClientModule.kt b/app/src/main/java/me/ash/reader/data/module/OkHttpClientModule.kt index 279248b..08e9c32 100644 --- a/app/src/main/java/me/ash/reader/data/module/OkHttpClientModule.kt +++ b/app/src/main/java/me/ash/reader/data/module/OkHttpClientModule.kt @@ -45,7 +45,6 @@ import javax.net.ssl.X509TrustManager @Module @InstallIn(SingletonComponent::class) object OkHttpClientModule { - @Provides @Singleton fun provideOkHttpClient( diff --git a/app/src/main/java/me/ash/reader/data/repository/RssHelper.kt b/app/src/main/java/me/ash/reader/data/repository/RssHelper.kt index 25528d0..7941255 100644 --- a/app/src/main/java/me/ash/reader/data/repository/RssHelper.kt +++ b/app/src/main/java/me/ash/reader/data/repository/RssHelper.kt @@ -3,6 +3,7 @@ package me.ash.reader.data.repository import android.content.Context import android.text.Html import android.util.Log +import com.rometools.rome.feed.synd.SyndEntry import com.rometools.rome.feed.synd.SyndFeed import com.rometools.rome.io.SyndFeedInput import com.rometools.rome.io.XmlReader @@ -20,7 +21,8 @@ import net.dankito.readability4j.Readability4J import net.dankito.readability4j.extended.Readability4JExtended import okhttp3.OkHttpClient import okhttp3.Request -import java.net.URL +import okhttp3.executeAsync +import java.io.InputStream import java.util.* import javax.inject.Inject @@ -35,7 +37,8 @@ class RssHelper @Inject constructor( suspend fun searchFeed(feedLink: String): FeedWithArticle { return withContext(dispatcherIO) { val accountId = context.currentAccountId - val parseRss: SyndFeed = SyndFeedInput().build(XmlReader(URL(feedLink))) + val parseRss: SyndFeed = + SyndFeedInput().build(XmlReader(inputStream(okHttpClient, feedLink))) val feed = Feed( id = accountId.spacerDollar(UUID.randomUUID().toString()), name = parseRss.title!!, @@ -43,7 +46,8 @@ class RssHelper @Inject constructor( groupId = "", accountId = accountId, ) - FeedWithArticle(feed, queryRssXml(feed)) + val list = parseRss.entries.map { article(feed, context.currentAccountId, it) } + FeedWithArticle(feed, list) } } @@ -57,9 +61,7 @@ class RssHelper @Inject constructor( @Throws(Exception::class) suspend fun parseFullContent(link: String, title: String): String { return withContext(dispatcherIO) { - val response = okHttpClient - .newCall(Request.Builder().url(link).build()) - .execute() + val response = response(okHttpClient, link) val content = response.body!!.string() val readability4J: Readability4J = Readability4JExtended(link, content) @@ -82,53 +84,54 @@ class RssHelper @Inject constructor( latestLink: String? = null, ): List
{ return withContext(dispatcherIO) { - val a = mutableListOf
() val accountId = context.currentAccountId val parseRss: SyndFeed = SyndFeedInput().build( - XmlReader(URL(feed.url).openConnection().apply { - connectTimeout = 5000 - readTimeout = 5000 - }) + XmlReader(inputStream(okHttpClient, feed.url)) ) - parseRss.entries.forEach { - if (latestLink != null && latestLink == it.link) return@withContext a - val desc = it.description?.value - val content = it.contents - .takeIf { it.isNotEmpty() } - ?.let { it.joinToString("\n") { it.value } } - Log.i( - "RLog", - "request rss:\n" + - "name: ${feed.name}\n" + - "feedUrl: ${feed.url}\n" + - "url: ${it.link}\n" + - "title: ${it.title}\n" + - "desc: ${desc}\n" + - "content: ${content}\n" - ) - a.add( - Article( - id = accountId.spacerDollar(UUID.randomUUID().toString()), - accountId = accountId, - feedId = feed.id, - date = it.publishedDate ?: it.updatedDate ?: Date(), - title = Html.fromHtml(it.title.toString()).toString(), - author = it.author, - rawDescription = (content ?: desc) ?: "", - shortDescription = (Readability4JExtended("", desc ?: content ?: "") - .parse().textContent ?: "") - .take(100) - .trim(), - fullContent = content, - img = findImg((content ?: desc) ?: ""), - link = it.link ?: "", - ) - ) - } - a + parseRss.entries.asSequence() + .takeWhile { latestLink == null || latestLink != it.link } + .map { article(feed, accountId, it) } + .toList() } } + private fun article( + feed: Feed, + accountId: Int, + syndEntry: SyndEntry + ): Article { + val desc = syndEntry.description?.value + val content = syndEntry.contents + .takeIf { it.isNotEmpty() } + ?.let { it.joinToString("\n") { it.value } } + Log.i( + "RLog", + "request rss:\n" + + "name: ${feed.name}\n" + + "feedUrl: ${feed.url}\n" + + "url: ${syndEntry.link}\n" + + "title: ${syndEntry.title}\n" + + "desc: ${desc}\n" + + "content: ${content}\n" + ) + return Article( + id = accountId.spacerDollar(UUID.randomUUID().toString()), + accountId = accountId, + feedId = feed.id, + date = syndEntry.publishedDate ?: syndEntry.updatedDate ?: Date(), + title = Html.fromHtml(syndEntry.title.toString()).toString(), + author = syndEntry.author, + rawDescription = (content ?: desc) ?: "", + shortDescription = (Readability4JExtended("", desc ?: content ?: "") + .parse().textContent ?: "") + .take(100) + .trim(), + fullContent = content, + img = findImg((content ?: desc) ?: ""), + link = syndEntry.link ?: "", + ) + } + private fun findImg(rawDescription: String): String? { // From: https://gitlab.com/spacecowboy/Feeder // Using negative lookahead to skip data: urls, being inline base64 @@ -146,9 +149,7 @@ class RssHelper @Inject constructor( ) { withContext(dispatcherIO) { val domainRegex = Regex("(http|https)://(www.)?(\\w+(\\.)?)+") - val request = OkHttpClient() - .newCall(Request.Builder().url(articleLink).build()) - .execute() + val request = response(okHttpClient, articleLink) val content = request.body!!.string() val regex = Regex("""