Threading Implementation for higher stability
This commit is contained in:
parent
8b3e6a8380
commit
211b44978a
5 changed files with 244 additions and 159 deletions
src/config
|
@ -1,11 +1,25 @@
|
|||
use std::{fs::{self, OpenOptions}, path::Path, io::Write, thread::sleep, time, error::Error};
|
||||
use std::{
|
||||
error::Error,
|
||||
fs::{self, OpenOptions},
|
||||
io::Write,
|
||||
path::Path,
|
||||
thread::sleep,
|
||||
time,
|
||||
};
|
||||
|
||||
use lemmy_api_common::{sensitive::Sensitive, post::CreatePost, community::{ListCommunities, ListCommunitiesResponse}};
|
||||
use lemmy_db_schema::{newtypes::{LanguageId, CommunityId, PostId}, ListingType};
|
||||
use lemmy_api_common::{
|
||||
community::{ListCommunities, ListCommunitiesResponse},
|
||||
post::CreatePost,
|
||||
sensitive::Sensitive,
|
||||
};
|
||||
use lemmy_db_schema::{
|
||||
newtypes::{CommunityId, LanguageId, PostId},
|
||||
ListingType,
|
||||
};
|
||||
use serde_derive::{Deserialize, Serialize};
|
||||
use url::Url;
|
||||
|
||||
use crate::{CLIENT};
|
||||
use crate::CLIENT;
|
||||
|
||||
macro_rules! pub_struct {
|
||||
($name:ident {$($field:ident: $t:ty,)*}) => {
|
||||
|
@ -40,7 +54,7 @@ impl Secrets {
|
|||
#[derive(Serialize, Deserialize, Clone, PartialEq)]
|
||||
pub(crate) struct LemmyLogin {
|
||||
pub(crate) username: String,
|
||||
password: String
|
||||
password: String,
|
||||
}
|
||||
|
||||
impl LemmyLogin {
|
||||
|
@ -49,7 +63,7 @@ impl LemmyLogin {
|
|||
}
|
||||
|
||||
pub(crate) fn get_password(&self) -> Sensitive<String> {
|
||||
return Sensitive::new(self.password.clone())
|
||||
return Sensitive::new(self.password.clone());
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -97,58 +111,58 @@ impl Config {
|
|||
self.reddit_config = config_parse.reddit_config;
|
||||
}
|
||||
|
||||
pub(crate) fn check_feeds(&mut self, post_history: &mut Vec<PrevPost>
|
||||
, community_ids: &CommunitiesVector, auth: &Sensitive<String>) -> Result<Vec<(CreatePost, (Option<usize>, usize, String))>, Box<dyn Error>> {
|
||||
pub(crate) async fn check_feeds(
|
||||
&mut self,
|
||||
post_history: &mut Vec<PrevPost>,
|
||||
community_ids: &CommunitiesVector,
|
||||
auth: &Sensitive<String>,
|
||||
) -> Result<Vec<(CreatePost, (Option<usize>, usize, String))>, Box<dyn Error>> {
|
||||
let mut post_queue: Vec<(CreatePost, (Option<usize>, usize, String))> = vec![];
|
||||
|
||||
match self.feeds.iter().map(|feed| {
|
||||
let mut i = 0;
|
||||
while i < self.feeds.len() {
|
||||
let feed = &self.feeds[i];
|
||||
|
||||
let res = CLIENT
|
||||
.get(feed.feed_url.clone())
|
||||
.send()?.text()?;
|
||||
.send()
|
||||
.await?
|
||||
.text()
|
||||
.await?;
|
||||
|
||||
let data: FeedData = serde_json::from_str(&res).unwrap();
|
||||
|
||||
let mut prev_post_idx: Option<usize> = None;
|
||||
let mut do_post = true;
|
||||
post_history
|
||||
.iter()
|
||||
.enumerate()
|
||||
.for_each(|(idx, post)| {
|
||||
if &post.last_post_url == &data.items[0].url {
|
||||
do_post = false;
|
||||
} else if &post.title == &data.title {
|
||||
prev_post_idx = Some(idx);
|
||||
}
|
||||
});
|
||||
post_history.iter().enumerate().for_each(|(idx, post)| {
|
||||
if &post.last_post_url == &data.items[0].url {
|
||||
do_post = false;
|
||||
} else if &post.title == &data.title {
|
||||
prev_post_idx = Some(idx);
|
||||
}
|
||||
});
|
||||
|
||||
if do_post {
|
||||
let item = &data.items[0];
|
||||
let new_post = CreatePost {
|
||||
name: item.title.clone(),
|
||||
community_id: community_ids.find(&feed.communities.chapter),
|
||||
url: Some(Url::parse(&item.url).unwrap()),
|
||||
body: Some(
|
||||
"[Reddit](https://reddit.com/r/HonzukinoGekokujou)\n\n[Discord](https://discord.com/invite/fGefmzu)".into(),
|
||||
),
|
||||
honeypot: None,
|
||||
nsfw: Some(false),
|
||||
language_id: Some(LanguageId(37)), // TODO get this id once every few hours per API request, the ordering of IDs suggests that the EN Id might change in the future
|
||||
auth: auth.clone(),
|
||||
};
|
||||
name: item.title.clone(),
|
||||
community_id: community_ids.find(&feed.communities.chapter),
|
||||
url: Some(Url::parse(&item.url).unwrap()),
|
||||
body: Some(
|
||||
"[Reddit](https://reddit.com/r/HonzukinoGekokujou)\n\n[Discord](https://discord.com/invite/fGefmzu)".into(),
|
||||
),
|
||||
honeypot: None,
|
||||
nsfw: Some(false),
|
||||
language_id: Some(LanguageId(37)), // TODO get this id once every few hours per API request, the ordering of IDs suggests that the EN Id might change in the future
|
||||
auth: auth.clone(),
|
||||
};
|
||||
|
||||
let prev_data = (
|
||||
prev_post_idx,
|
||||
feed.id,
|
||||
data.title
|
||||
);
|
||||
let prev_data = (prev_post_idx, feed.id, data.title);
|
||||
|
||||
post_queue.push((new_post, prev_data));
|
||||
}
|
||||
sleep(time::Duration::from_millis(100)); // Should prevent dos-ing J-Novel servers
|
||||
return Ok(());
|
||||
}).collect() {
|
||||
Ok(()) => {}
|
||||
Err(e) => return Err(e)
|
||||
i += 1;
|
||||
}
|
||||
|
||||
return Ok(post_queue);
|
||||
|
@ -179,7 +193,7 @@ pub(crate) enum LemmyCommunities {
|
|||
aobprepub,
|
||||
aoblightnovel,
|
||||
aobmanga,
|
||||
metadiscussions
|
||||
metadiscussions,
|
||||
}
|
||||
|
||||
pub_struct!(FeedRedditSettings {
|
||||
|
@ -198,7 +212,7 @@ pub_struct!(PrevPost {
|
|||
impl PrevPost {
|
||||
pub(crate) fn load() -> Vec<PrevPost> {
|
||||
let history;
|
||||
|
||||
|
||||
if Path::new("posts.json").exists() {
|
||||
let file_contents = match fs::read_to_string("posts.json") {
|
||||
Ok(data) => data,
|
||||
|
@ -211,12 +225,10 @@ impl PrevPost {
|
|||
Err(e) => panic!("ERROR: posts.json could not be parsed:\n\n{:#?}", e),
|
||||
};
|
||||
history = history_parse;
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
history = [].to_vec()
|
||||
}
|
||||
}
|
||||
else {
|
||||
} else {
|
||||
let _ = fs::File::create("posts.json");
|
||||
history = [].to_vec()
|
||||
}
|
||||
|
@ -225,7 +237,12 @@ impl PrevPost {
|
|||
}
|
||||
|
||||
pub(crate) fn save(data: &Vec<PrevPost>) {
|
||||
let mut file = OpenOptions::new().read(true).write(true).create(true).open("posts.json").unwrap();
|
||||
let mut file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.open("posts.json")
|
||||
.unwrap();
|
||||
|
||||
let json_data = serde_json::to_string_pretty(&data).unwrap();
|
||||
|
||||
|
@ -243,9 +260,7 @@ pub_struct!(FeedData {
|
|||
items: Vec<FeedEntry>,
|
||||
});
|
||||
|
||||
pub_struct!(FeedAuthor {
|
||||
name: String,
|
||||
});
|
||||
pub_struct!(FeedAuthor { name: String, });
|
||||
|
||||
pub_struct!(FeedEntry {
|
||||
id: String,
|
||||
|
@ -255,7 +270,7 @@ pub_struct!(FeedEntry {
|
|||
image: Option<String>,
|
||||
date_published: String,
|
||||
});
|
||||
|
||||
|
||||
// Bot Helper Structs
|
||||
pub_struct!(CommunitiesVector {
|
||||
ids: Vec<(CommunityId, String)>,
|
||||
|
@ -263,11 +278,15 @@ pub_struct!(CommunitiesVector {
|
|||
|
||||
impl CommunitiesVector {
|
||||
pub(crate) fn new() -> CommunitiesVector {
|
||||
CommunitiesVector{ids: vec![]}
|
||||
CommunitiesVector { ids: vec![] }
|
||||
}
|
||||
|
||||
#[warn(unused_results)]
|
||||
pub(crate) fn load(&mut self, auth: &Sensitive<String>, base: &String) -> Result<(), Box<dyn Error>> {
|
||||
pub(crate) async fn load(
|
||||
&mut self,
|
||||
auth: &Sensitive<String>,
|
||||
base: &String,
|
||||
) -> Result<(), Box<dyn Error>> {
|
||||
let params = ListCommunities {
|
||||
auth: Some(auth.clone()),
|
||||
type_: Some(ListingType::Local),
|
||||
|
@ -277,7 +296,10 @@ impl CommunitiesVector {
|
|||
let res = CLIENT
|
||||
.get(base.clone() + "/api/v3/community/list")
|
||||
.query(¶ms)
|
||||
.send()?.text()?;
|
||||
.send()
|
||||
.await?
|
||||
.text()
|
||||
.await?;
|
||||
|
||||
let site_data: ListCommunitiesResponse = serde_json::from_str(&res).unwrap();
|
||||
|
||||
|
@ -301,6 +323,6 @@ impl CommunitiesVector {
|
|||
ret_id = id.0;
|
||||
}
|
||||
});
|
||||
return ret_id;
|
||||
return ret_id;
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue