Neshura/aob-lemmy-bot
Neshura/aob-lemmy-bot
1
0
Fork 0
Code Issues9 Pull requests Projects1 Releases70 Packages2 Wiki Activity Actions

Threading Implementation for higher stability

Browse source
This commit is contained in:
Neshura 2023-08-31 23:36:37 +02:00
parent 8b3e6a8380
commit 211b44978a
Signed by: Neshura
GPG key ID: B6983AAA6B9A7A6C
5 changed files with 244 additions and 159 deletions
src/config
mod.rs

132
src/config/mod.rs
View file

@ -1,11 +1,25 @@
use std::{fs::{self, OpenOptions}, path::Path, io::Write, thread::sleep, time, error::Error};
use std::{
error::Error,
fs::{self, OpenOptions},
io::Write,
path::Path,
thread::sleep,
time,
};
use lemmy_api_common::{sensitive::Sensitive, post::CreatePost, community::{ListCommunities, ListCommunitiesResponse}};
use lemmy_db_schema::{newtypes::{LanguageId, CommunityId, PostId}, ListingType};
use lemmy_api_common::{
community::{ListCommunities, ListCommunitiesResponse},
post::CreatePost,
sensitive::Sensitive,
};
use lemmy_db_schema::{
newtypes::{CommunityId, LanguageId, PostId},
ListingType,
};
use serde_derive::{Deserialize, Serialize};
use url::Url;
use crate::{CLIENT};
use crate::CLIENT;
macro_rules! pub_struct {
($name:ident {$($field:ident: $t:ty,)*}) => {
@ -40,7 +54,7 @@ impl Secrets {
#[derive(Serialize, Deserialize, Clone, PartialEq)]
pub(crate) struct LemmyLogin {
pub(crate) username: String,
password: String
password: String,
}
impl LemmyLogin {
@ -49,7 +63,7 @@ impl LemmyLogin {
}
pub(crate) fn get_password(&self) -> Sensitive<String> {
return Sensitive::new(self.password.clone())
return Sensitive::new(self.password.clone());
}
}
@ -97,58 +111,58 @@ impl Config {
self.reddit_config = config_parse.reddit_config;
}
pub(crate) fn check_feeds(&mut self, post_history: &mut Vec<PrevPost>
, community_ids: &CommunitiesVector, auth: &Sensitive<String>) -> Result<Vec<(CreatePost, (Option<usize>, usize, String))>, Box<dyn Error>> {
pub(crate) async fn check_feeds(
&mut self,
post_history: &mut Vec<PrevPost>,
community_ids: &CommunitiesVector,
auth: &Sensitive<String>,
) -> Result<Vec<(CreatePost, (Option<usize>, usize, String))>, Box<dyn Error>> {
let mut post_queue: Vec<(CreatePost, (Option<usize>, usize, String))> = vec![];
match self.feeds.iter().map(|feed| {
let mut i = 0;
while i < self.feeds.len() {
let feed = &self.feeds[i];
let res = CLIENT
.get(feed.feed_url.clone())
.send()?.text()?;
.send()
.await?
.text()
.await?;
let data: FeedData = serde_json::from_str(&res).unwrap();
let mut prev_post_idx: Option<usize> = None;
let mut do_post = true;
post_history
.iter()
.enumerate()
.for_each(|(idx, post)| {
if &post.last_post_url == &data.items[0].url {
do_post = false;
} else if &post.title == &data.title {
prev_post_idx = Some(idx);
}
});
post_history.iter().enumerate().for_each(|(idx, post)| {
if &post.last_post_url == &data.items[0].url {
do_post = false;
} else if &post.title == &data.title {
prev_post_idx = Some(idx);
}
});
if do_post {
let item = &data.items[0];
let new_post = CreatePost {
name: item.title.clone(),
community_id: community_ids.find(&feed.communities.chapter),
url: Some(Url::parse(&item.url).unwrap()),
body: Some(
"[Reddit](https://reddit.com/r/HonzukinoGekokujou)\n\n[Discord](https://discord.com/invite/fGefmzu)".into(),
),
honeypot: None,
nsfw: Some(false),
language_id: Some(LanguageId(37)), // TODO get this id once every few hours per API request, the ordering of IDs suggests that the EN Id might change in the future
auth: auth.clone(),
};
name: item.title.clone(),
community_id: community_ids.find(&feed.communities.chapter),
url: Some(Url::parse(&item.url).unwrap()),
body: Some(
"[Reddit](https://reddit.com/r/HonzukinoGekokujou)\n\n[Discord](https://discord.com/invite/fGefmzu)".into(),
),
honeypot: None,
nsfw: Some(false),
language_id: Some(LanguageId(37)), // TODO get this id once every few hours per API request, the ordering of IDs suggests that the EN Id might change in the future
auth: auth.clone(),
};
let prev_data = (
prev_post_idx,
feed.id,
data.title
);
let prev_data = (prev_post_idx, feed.id, data.title);
post_queue.push((new_post, prev_data));
}
sleep(time::Duration::from_millis(100)); // Should prevent dos-ing J-Novel servers
return Ok(());
}).collect() {
Ok(()) => {}
Err(e) => return Err(e)
i += 1;
}
return Ok(post_queue);
@ -179,7 +193,7 @@ pub(crate) enum LemmyCommunities {
aobprepub,
aoblightnovel,
aobmanga,
metadiscussions
metadiscussions,
}
pub_struct!(FeedRedditSettings {
@ -198,7 +212,7 @@ pub_struct!(PrevPost {
impl PrevPost {
pub(crate) fn load() -> Vec<PrevPost> {
let history;
if Path::new("posts.json").exists() {
let file_contents = match fs::read_to_string("posts.json") {
Ok(data) => data,
@ -211,12 +225,10 @@ impl PrevPost {
Err(e) => panic!("ERROR: posts.json could not be parsed:\n\n{:#?}", e),
};
history = history_parse;
}
else {
} else {
history = [].to_vec()
}
}
else {
} else {
let _ = fs::File::create("posts.json");
history = [].to_vec()
}
@ -225,7 +237,12 @@ impl PrevPost {
}
pub(crate) fn save(data: &Vec<PrevPost>) {
let mut file = OpenOptions::new().read(true).write(true).create(true).open("posts.json").unwrap();
let mut file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.open("posts.json")
.unwrap();
let json_data = serde_json::to_string_pretty(&data).unwrap();
@ -243,9 +260,7 @@ pub_struct!(FeedData {
items: Vec<FeedEntry>,
});
pub_struct!(FeedAuthor {
name: String,
});
pub_struct!(FeedAuthor { name: String, });
pub_struct!(FeedEntry {
id: String,
@ -255,7 +270,7 @@ pub_struct!(FeedEntry {
image: Option<String>,
date_published: String,
});
// Bot Helper Structs
pub_struct!(CommunitiesVector {
ids: Vec<(CommunityId, String)>,
@ -263,11 +278,15 @@ pub_struct!(CommunitiesVector {
impl CommunitiesVector {
pub(crate) fn new() -> CommunitiesVector {
CommunitiesVector{ids: vec![]}
CommunitiesVector { ids: vec![] }
}
#[warn(unused_results)]
pub(crate) fn load(&mut self, auth: &Sensitive<String>, base: &String) -> Result<(), Box<dyn Error>> {
pub(crate) async fn load(
&mut self,
auth: &Sensitive<String>,
base: &String,
) -> Result<(), Box<dyn Error>> {
let params = ListCommunities {
auth: Some(auth.clone()),
type_: Some(ListingType::Local),
@ -277,7 +296,10 @@ impl CommunitiesVector {
let res = CLIENT
.get(base.clone() + "/api/v3/community/list")
.query(&params)
.send()?.text()?;
.send()
.await?
.text()
.await?;
let site_data: ListCommunitiesResponse = serde_json::from_str(&res).unwrap();
@ -301,6 +323,6 @@ impl CommunitiesVector {
ret_id = id.0;
}
});
return ret_id;
return ret_id;
}
}