From b36bc9b0b2a40208a7fffb65dd57f39567003f01 Mon Sep 17 00:00:00 2001
From: Neshura <neshura@neshweb.net>
Date: Tue, 18 Feb 2025 22:17:29 +0100
Subject: [PATCH] Add Creator Info + Fetching to JNC Module. Implement .cbz
chapter generation
---
jnc/jnc.go | 51 +++++++-
main.go | 343 +++++++++++++++++++++++++++++++++++++++++++++--------
2 files changed, 345 insertions(+), 49 deletions(-)
diff --git a/jnc/jnc.go b/jnc/jnc.go
index 5f0a094..38a121f 100644
--- a/jnc/jnc.go
+++ b/jnc/jnc.go
@@ -8,6 +8,7 @@ import (
"io"
"net/http"
"os"
+ "slices"
"sort"
"strconv"
"strings"
@@ -90,7 +91,7 @@ type Volume struct {
Number int `json:"number"`
OriginalPublisher string `json:"originalPublisher"`
Label string `json:"label"`
- Creators []string `json:"creators"` // TODO: check, might not be correct
+ Creators Creators `json:"creators"` // TODO: check, might not be correct
hidden bool
ForumTopicId int `json:"forumTopicId"`
Created string `json:"created"`
@@ -105,6 +106,28 @@ type Volume struct {
OnSale bool `json:"onSale"`
}
+type Creators []Creator
+
+type Creator struct {
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Role string `json:"role"`
+ OriginalName string `json:"originalName "`
+}
+
+func (creators *Creators) Contains(name string) bool {
+ return slices.ContainsFunc(*creators, func(c Creator) bool {
+ return c.Name == name
+ })
+}
+
+func (creators *Creators) Get(name string) *Creator {
+ idx := slices.IndexFunc(*creators, func(c Creator) bool {
+ return c.Name == name
+ })
+ return &(*creators)[idx]
+}
+
type VolumeAugmented struct {
Info Volume
Downloads []Download
@@ -116,7 +139,7 @@ func (volume *VolumeAugmented) UpdateAvailable() bool {
if volume.downloaded == "" && len(volume.Downloads) != 0 {
return true // The file has never been downloaded but at least one is available
}
-
+
downloadedTime, err := time.Parse(time.RFC3339, volume.downloaded)
if err != nil {
panic(err)
@@ -367,6 +390,30 @@ func (jncApi *Api) FetchLibrarySeries() error {
return nil
}
+// FetchVolumeInfo retrieves additional Volume Info that was not returned when retrieving the entire Library
+func (jncApi *Api) FetchVolumeInfo(volume Volume) (Volume, error) {
+ fmt.Println("Fetching Volume details...")
+ res, err := http.Get(ApiV2Url + "volumes/" + volume.Id + "?" + jncApi.ReturnFormat() + "&" + jncApi.AuthParam())
+ if err != nil {
+ return volume, err
+ }
+
+ if res.StatusCode != 200 {
+ return volume, errors.New(res.Status)
+ }
+
+ body, err := io.ReadAll(res.Body)
+ if err != nil {
+ return volume, err
+ }
+ err = json.Unmarshal(body, &volume)
+ if err != nil {
+ return volume, err
+ }
+
+ return volume, nil
+}
+
// GetLibrarySeries returns a list of unique series found in the User's library. Tags are only included if FetchLibrarySeries was previously called.
func (jncApi *Api) GetLibrarySeries() (seriesList []SerieAugmented, err error) {
arr := make([]SerieAugmented, 0, len(jncApi._series))
diff --git a/main.go b/main.go
index f05a8ad..1e3e21a 100644
--- a/main.go
+++ b/main.go
@@ -3,7 +3,9 @@ package main
import (
"archive/zip"
"bufio"
+ "bytes"
"container/list"
+ "encoding/xml"
"fmt"
"github.com/beevik/etree"
"io"
@@ -14,6 +16,7 @@ import (
"slices"
"strconv"
"strings"
+ "time"
)
type Chapter struct {
@@ -23,6 +26,7 @@ type Chapter struct {
title string
firstPage string
lastPage string
+ pages []string
}
type FileWrapper struct {
@@ -64,7 +68,25 @@ func ClearScreen() {
}
func main() {
- downloadDir := "/home/neshura/Documents/Go Workspaces/Kavita Helper/"
+ interactive := false
+ if slices.Contains(os.Args, "-I") {
+ interactive = true
+ }
+
+ if !interactive {
+ panic("automatic mode is not implemented yet")
+ }
+
+ var downloadDir string
+ if slices.Contains(os.Args, "-d") {
+ idx := slices.Index(os.Args, "-d")
+ downloadDir = os.Args[idx+1]
+ if strings.LastIndex(downloadDir, "/") != len(downloadDir)-1 {
+ downloadDir = downloadDir + "/"
+ }
+ } else {
+ panic("working directory not specified")
+ }
// initialize the J-Novel Club Instance
jnovel := jnc.NewJNC()
@@ -192,10 +214,16 @@ func main() {
func HandleSeries(jnovel jnc.Api, serie jnc.SerieAugmented, downloadDir string, updatedOnly bool) {
for v := range serie.Volumes {
volume := serie.Volumes[v]
- if len(volume.Downloads) != 0 && volume.UpdateAvailable() {
+ if updatedOnly {
+ if len(volume.Downloads) != 0 && volume.UpdateAvailable() {
+ downloadDir = PrepareSerieDirectory(serie, volume, downloadDir)
+ HandleVolume(jnovel, serie, volume, downloadDir)
+ }
+ } else {
downloadDir = PrepareSerieDirectory(serie, volume, downloadDir)
HandleVolume(jnovel, serie, volume, downloadDir)
}
+
}
}
@@ -300,7 +328,6 @@ func DownloadAndProcessEpub(jnovel jnc.Api, serie jnc.SerieAugmented, volume jnc
}
if metadata.fileSet && navigation.fileSet {
- println("breaking")
break
}
}
@@ -310,11 +337,104 @@ func DownloadAndProcessEpub(jnovel jnc.Api, serie jnc.SerieAugmented, volume jnc
switch FormatMetadataName[metadata.fileName] {
case "manga":
{
+ comicInfoName := "ComicInfo.xml"
chapterList := GenerateMangaChapterList(navigation)
+ if chapterList.Len() == 0 {
+ fmt.Println("No chapters found, chapter name likely not supported")
+ }
+
+ basePath := downloadDirectory + volume.Info.Title + "/"
+ PrepareVolumeDirectory(basePath)
+ volume.Info, err = jnovel.FetchVolumeInfo(volume.Info)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ doc := etree.NewDocument()
+ reader, err := metadata.file.Open()
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ if _, err := doc.ReadFrom(reader); err != nil {
+ _ = reader.Close()
+ panic(err)
+ }
+
+ language := doc.FindElement("package/metadata/dc:language").Text()
+
for ch := chapterList.Front(); ch != nil; ch = ch.Next() {
chap := ch.Value.(Chapter)
- fmt.Printf("[%s] %s: %s - %s\n", chap.chDisplay, chap.title, chap.firstPage, chap.lastPage)
+
+ zipPath := basePath + "Chapter " + chap.chDisplay + ".cbz"
+
+ newZipFile, err := os.Create(zipPath)
+ if err != nil {
+ panic(err)
+ }
+
+ newZip := zip.NewWriter(newZipFile)
+
+ for chapterPageIndex := range chap.pages {
+ chapterFile := chap.pages[chapterPageIndex]
+
+ epubFileIndex := slices.IndexFunc(epub.File, func(f *zip.File) bool {
+ fParts := strings.Split(f.Name, "/")
+ cParts := strings.Split(chapterFile, "/")
+ return fParts[len(fParts)-1] == cParts[len(cParts)-1]
+ })
+
+ epubFile := epub.File[epubFileIndex]
+
+ doc := etree.NewDocument()
+ reader, err := epubFile.Open()
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ if _, err := doc.ReadFrom(reader); err != nil {
+ _ = reader.Close()
+ panic(err)
+ }
+
+ imageFilePath := doc.FindElement("//html/body/svg/image").SelectAttr("href").Value
+ _ = reader.Close()
+
+ if imageFilePath[2:] == ".." {
+ fParts := strings.Split(epubFile.Name, "/")
+ imageFilePath = fParts[len(fParts)-2] + imageFilePath[2:len(imageFilePath)-1]
+ }
+
+ iParts := strings.Split(imageFilePath, "/")
+ imageFileIndex := slices.IndexFunc(epub.File, func(f *zip.File) bool {
+ fParts := strings.Split(f.Name, "/")
+ return fParts[len(fParts)-1] == iParts[len(iParts)-1]
+ })
+
+ imageFile := epub.File[imageFileIndex]
+ fileName := fmt.Sprintf("%020s", chapterPageIndex+1) + "." + strings.Split(iParts[len(iParts)-1], ".")[1]
+
+ err = addFileToZip(newZip, imageFile, fileName)
+ if err != nil {
+ fmt.Println(err)
+ }
+ }
+
+ comicInfo, err := GenerateChapterMetadata(volume, serie, len(chap.pages), language)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ err = addBytesToZip(newZip, comicInfo, comicInfoName)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ epub.Close()
+ newZip.Close()
+ newZipFile.Close()
+ os.Remove(file)
}
}
case "novel":
@@ -416,65 +536,193 @@ func DownloadAndProcessEpub(jnovel jnc.Api, serie jnc.SerieAugmented, volume jnc
}
}
+func GenerateChapterMetadata(volume jnc.VolumeAugmented, serie jnc.SerieAugmented, pageCount int, language string) ([]byte, error) {
+ comicInfo := ComicInfo{
+ XMLName: "ComicInfo",
+ XMLNS: "http://www.w3.org/2001/XMLSchema-instance",
+ XSI: "ComicInfo.xsd",
+ }
+
+ vInfo := volume.Info
+ sInfo := serie.Info
+
+ comicInfo.Series = sInfo.Title
+ comicInfo.Number = vInfo.Number
+
+ comicInfo.Count = -1 // TODO somehow fetch actual completion status
+
+ comicInfo.Summary = vInfo.Description
+
+ publishingTime, err := time.Parse(time.RFC3339, vInfo.Publishing)
+ if err != nil {
+ return nil, err
+ }
+ comicInfo.Year = publishingTime.Year()
+ comicInfo.Month = int(publishingTime.Month())
+ comicInfo.Day = publishingTime.Day()
+
+ if vInfo.Creators.Contains("AUTHOR") {
+ comicInfo.Writer = vInfo.Creators.Get("AUTHOR").Name
+ }
+
+ if vInfo.Creators.Contains("ILLUSTRATOR") {
+ comicInfo.Letterer = vInfo.Creators.Get("ILLUSTRATOR").Name
+ comicInfo.CoverArtist = vInfo.Creators.Get("ILLUSTRATOR").Name
+ }
+
+ if vInfo.Creators.Contains("TRANSLATOR") {
+ comicInfo.Translator = vInfo.Creators.Get("TRANSLATOR").Name
+ }
+
+ if vInfo.Creators.Contains("EDITOR") {
+ comicInfo.Editor = vInfo.Creators.Get("EDITOR").Name
+ }
+
+ if vInfo.Creators.Contains("PUBLISHER") {
+ comicInfo.Publisher = vInfo.Creators.Get("PUBLISHER").Name
+ }
+
+ comicInfo.Tags = strings.Join(sInfo.Tags, ",")
+ comicInfo.PageCount = pageCount
+ comicInfo.Language = language
+ comicInfo.Format = "Comic" // JNC reports this as type in the epub file
+
+ return xml.Marshal(comicInfo)
+}
+
+type ComicInfo struct {
+ XMLName string `xml:"ComicInfo"`
+ XMLNS string `xml:"xmlns,attr"`
+ XSI string `xml:"xsi,attr"`
+ Series string `xml:"Series"`
+ Number int `xml:"Number"`
+ Count int `xml:"Count"`
+ Summary string `xml:"Summary"`
+ Year int `xml:"Year"`
+ Month int `xml:"Month"`
+ Day int `xml:"Day"`
+ Writer string `xml:"Writer"`
+ Letterer string `xml:"Letterer"`
+ CoverArtist string `xml:"CoverArtist"`
+ Editor string `xml:"Editor"`
+ Translator string `xml:"Translator"`
+ Publisher string `xml:"Publisher"`
+ Tags string `xml:"Tags"`
+ PageCount int `xml:"PageCount"`
+ Language string `xml:"LanguageISO"`
+ Format string `xml:"Format"`
+}
+
+func addBytesToZip(zipWriter *zip.Writer, fileBytes []byte, filename string) error {
+ reader := bytes.NewReader(fileBytes)
+
+ w, err := zipWriter.Create(filename)
+ if err != nil {
+ return error(err)
+ }
+
+ if _, err := io.Copy(w, reader); err != nil {
+ return error(err)
+ }
+
+ return nil
+}
+
+func addFileToZip(zipWriter *zip.Writer, file *zip.File, filename string) error {
+ fileToZip, err := file.Open()
+ if err != nil {
+ return error(err)
+ }
+ defer fileToZip.Close()
+
+ w, err := zipWriter.Create(filename)
+ if err != nil {
+ return error(err)
+ }
+
+ if _, err := io.Copy(w, fileToZip); err != nil {
+ return error(err)
+ }
+
+ return nil
+}
+
+func PrepareVolumeDirectory(directory string) {
+ _, err := os.Stat(directory)
+ if err != nil {
+ err = os.Mkdir(directory, 0755)
+ if err != nil {
+ panic(err)
+ }
+ }
+}
+
func FinalizeChapters(chapters *list.List, pageList []*etree.Element) *list.List {
fmt.Println("Finalizing Chapters")
- lastPage := pageList[len(pageList)-1].SelectAttr("src").Value
-
sortedChapters := list.New()
+ var mainChapter Chapter
+
for ch := chapters.Back(); ch != nil; ch = ch.Prev() {
- var newChapterData Chapter
currentChapter := ch.Value.(Chapter)
if currentChapter.numberMain != -1 {
- newChapterData.title = currentChapter.title
- newChapterData.firstPage = currentChapter.firstPage
+ // pages
+ if currentChapter.numberSub != 0 {
+ numberMain, numberSub := AnalyzeMainChapter(ch)
+ currentChapter.numberMain = numberMain
+ currentChapter.numberSub = numberSub
+ currentChapter.chDisplay = strconv.FormatInt(int64(currentChapter.numberMain), 10) + "." + strconv.FormatInt(int64(currentChapter.numberSub), 10)
+ } else {
+ currentChapter.chDisplay = strconv.FormatInt(int64(currentChapter.numberMain), 10)
+ }
+
+ currentChapter.lastPage = pageList[len(pageList)-1].SelectAttr("src").Value
+ collecting := false
+ for p := range pageList {
+ page := pageList[p].SelectAttr("src").Value
+ if page == currentChapter.firstPage {
+ collecting = true
+ currentChapter.pages = append(currentChapter.pages, page)
+ } else if ch.Next() != nil && page == ch.Next().Value.(Chapter).firstPage {
+ currentChapter.lastPage = pageList[p-1].SelectAttr("src").Value
+ collecting = false
+ } else if collecting {
+ currentChapter.pages = append(currentChapter.pages, page)
+ }
+ }
+
+ sortedChapters.PushFront(currentChapter)
} else {
- mainChapter := GetLastValidChapterNumber(ch)
+ mainChapter = GetLastValidChapterNumber(ch)
+
for sch := sortedChapters.Front(); sch != nil; sch = sch.Next() {
if sch.Value.(Chapter).title == mainChapter.title {
- newChapterData = sch.Value.(Chapter)
- newChapterData.firstPage = currentChapter.firstPage
- sch.Value = newChapterData
+ mainChapterData := sch.Value.(Chapter)
+ mainChapterData.firstPage = currentChapter.firstPage
+
+ collecting := false
+ pages := []string{}
+ for p := range pageList {
+ page := pageList[p].SelectAttr("src").Value
+ if page == currentChapter.firstPage {
+ collecting = true
+ pages = append(pages, page)
+ } else if ch.Next() != nil && page == ch.Next().Value.(Chapter).firstPage {
+ currentChapter.lastPage = pageList[p-1].SelectAttr("src").Value
+ collecting = false
+ } else if collecting {
+ pages = append(pages, page)
+ }
+ }
+ mainChapterData.pages = append(pages, mainChapterData.pages...)
+ sch.Value = mainChapterData
break
}
}
- continue
}
-
- if currentChapter.numberSub != 0 {
- numberMain, numberSub := AnalyzeMainChapter(ch)
-
- newChapterData.numberMain = numberMain
- newChapterData.numberSub = numberSub
- } else {
- newChapterData.numberMain = currentChapter.numberMain
- newChapterData.numberSub = currentChapter.numberSub
- }
-
- if ch.Next() == nil {
- newChapterData.lastPage = lastPage
- } else {
- var lastChapterPage string
- for p := range pageList {
- page := pageList[p]
- if page.SelectAttr("src").Value == ch.Next().Value.(Chapter).firstPage {
- lastChapterPage = pageList[p-1].SelectAttr("src").Value
- }
- }
- newChapterData.lastPage = lastChapterPage
- }
-
- if newChapterData.numberSub != 0 {
- newChapterData.chDisplay = strconv.FormatInt(int64(newChapterData.numberMain), 10) + "." + strconv.FormatInt(int64(newChapterData.numberSub), 10)
- } else {
- newChapterData.chDisplay = strconv.FormatInt(int64(newChapterData.numberMain), 10)
- }
-
- sortedChapters.PushFront(newChapterData)
}
-
return sortedChapters
}
@@ -563,7 +811,7 @@ func GenerateMangaChapterList(navigation FileWrapper) *list.List {
label := navChapter.FindElement("./navLabel/text").Text()
page := navChapter.FindElement("./content")
- regex := "Chapter ([0-9]*)"
+ regex := "(?:Chapter|Episode) ([0-9]*)"
match, _ := regexp.MatchString(regex, label)
if match {
@@ -587,6 +835,7 @@ func GenerateMangaChapterList(navigation FileWrapper) *list.List {
title: label,
firstPage: page.SelectAttr("src").Value,
lastPage: "",
+ pages: []string{},
}
chapters.PushBack(chapterData)