1209 toDB

novel-demo/src/main/java/com/sf/controller/BookChapterController.java

@@ -0,0 +1,18 @@
+package com.sf.controller;
+
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.stereotype.Controller;
+
+/**
+ * <p>
+ * 小说章节 前端控制器
+ * </p>
+ *
+ * @author Qing
+ * @since 2023-12-09
+ */
+@Controller
+@RequestMapping("/bookChapter")
+public class BookChapterController {
+
+}

novel-demo/src/main/java/com/sf/mapper/BookChapterMapper.java

@@ -0,0 +1,16 @@
+package com.sf.mapper;
+
+import com.sf.po.BookChapter;
+import com.baomidou.mybatisplus.core.mapper.BaseMapper;
+
+/**
+ * <p>
+ * 小说章节 Mapper 接口
+ * </p>
+ *
+ * @author Qing
+ * @since 2023-12-09
+ */
+public interface BookChapterMapper extends BaseMapper<BookChapter> {
+
+}

novel-demo/src/main/java/com/sf/po/BookChapter.java

@@ -0,0 +1,126 @@
+package com.sf.po;
+
+import com.baomidou.mybatisplus.annotation.IdType;
+import com.baomidou.mybatisplus.annotation.TableId;
+import com.baomidou.mybatisplus.annotation.TableName;
+import java.io.Serializable;
+import java.time.LocalDateTime;
+import io.swagger.annotations.ApiModel;
+import io.swagger.annotations.ApiModelProperty;
+import lombok.Builder;
+
+/**
+ * <p>
+ * 小说章节
+ * </p>
+ *
+ * @author Qing
+ * @since 2023-12-09
+ */
+@Builder
+@TableName("book_chapter")
+@ApiModel(value = "BookChapter对象", description = "小说章节")
+public class BookChapter implements Serializable {
+
+    private static final long serialVersionUID = 1L;
+
+    @TableId(value = "id", type = IdType.AUTO)
+    private Long id;
+
+    @ApiModelProperty("小说ID")
+    private Long bookId;
+
+    @ApiModelProperty("章节号")
+    private Integer chapterNum;
+
+    @ApiModelProperty("章节名")
+    private String chapterName;
+
+    @ApiModelProperty("章节字数")
+    private Integer wordCount;
+
+    @ApiModelProperty("是否收费;1-收费 0-免费")
+    private Byte isVip;
+
+    private LocalDateTime createTime;
+
+    private LocalDateTime updateTime;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public Long getBookId() {
+        return bookId;
+    }
+
+    public void setBookId(Long bookId) {
+        this.bookId = bookId;
+    }
+
+    public Integer getChapterNum() {
+        return chapterNum;
+    }
+
+    public void setChapterNum(Integer chapterNum) {
+        this.chapterNum = chapterNum;
+    }
+
+    public String getChapterName() {
+        return chapterName;
+    }
+
+    public void setChapterName(String chapterName) {
+        this.chapterName = chapterName;
+    }
+
+    public Integer getWordCount() {
+        return wordCount;
+    }
+
+    public void setWordCount(Integer wordCount) {
+        this.wordCount = wordCount;
+    }
+
+    public Byte getIsVip() {
+        return isVip;
+    }
+
+    public void setIsVip(Byte isVip) {
+        this.isVip = isVip;
+    }
+
+    public LocalDateTime getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(LocalDateTime createTime) {
+        this.createTime = createTime;
+    }
+
+    public LocalDateTime getUpdateTime() {
+        return updateTime;
+    }
+
+    public void setUpdateTime(LocalDateTime updateTime) {
+        this.updateTime = updateTime;
+    }
+
+    @Override
+    public String toString() {
+        return "BookChapter{" +
+            "id = " + id +
+            ", bookId = " + bookId +
+            ", chapterNum = " + chapterNum +
+            ", chapterName = " + chapterName +
+            ", wordCount = " + wordCount +
+            ", isVip = " + isVip +
+            ", createTime = " + createTime +
+            ", updateTime = " + updateTime +
+        "}";
+    }
+}

novel-demo/src/main/java/com/sf/po/BookContent.java

@@ -7,6 +7,7 @@ import java.io.Serializable;
 import java.time.LocalDateTime;
 import io.swagger.annotations.ApiModel;
 import io.swagger.annotations.ApiModelProperty;
+import lombok.Builder;
 
 /**
  * <p>
@@ -16,6 +17,7 @@ import io.swagger.annotations.ApiModelProperty;
  * @author Qing
  * @since 2023-12-02
  */
+@Builder
 @TableName("book_content")
 @ApiModel(value = "BookContent对象", description = "小说内容")
 public class BookContent implements Serializable {

novel-demo/src/main/java/com/sf/po/BookInfo.java

@@ -3,112 +3,248 @@ package com.sf.po;
 import com.baomidou.mybatisplus.annotation.IdType;
 import com.baomidou.mybatisplus.annotation.TableId;
 import com.baomidou.mybatisplus.annotation.TableName;
-import lombok.Data;
 
+import java.io.Serializable;
 import java.time.LocalDateTime;
 
-@Data
+
+/**
+ * <p>
+ * 小说信息
+ * </p>
+ *
+ * @author Qing
+ * @since 2023-12-01
+ */
 @TableName("book_info")
-public class BookInfo {
+public class BookInfo implements Serializable {
+
+    private static final long serialVersionUID = 1L;
 
-    /**
-     * 主键
-     */
     @TableId(value = "id", type = IdType.AUTO)
     private Long id;
 
-    /**
-     * 作品方向;0-男频 1-女频
-     */
-    private Integer workDirection;
+    private Byte workDirection;
 
-    /**
-     * 类别ID
-     */
     private Long categoryId;
 
-    /**
-     * 类别名
-     */
     private String categoryName;
 
-    /**
-     * 小说封面地址
-     */
     private String picUrl;
 
-    /**
-     * 小说名
-     */
     private String bookName;
 
-    /**
-     * 作家id
-     */
     private Long authorId;
 
-    /**
-     * 作家名
-     */
     private String authorName;
 
-    /**
-     * 书籍描述
-     */
     private String bookDesc;
 
-    /**
-     * 评分;总分:10 ，真实评分 = score/10
-     */
-    private Integer score;
+    private Byte score;
 
-    /**
-     * 书籍状态;0-连载中 1-已完结
-     */
-    private Integer bookStatus;
+    private Byte bookStatus;
 
-    /**
-     * 点击量
-     */
     private Long visitCount;
 
-    /**
-     * 总字数
-     */
     private Integer wordCount;
 
-    /**
-     * 评论数
-     */
     private Integer commentCount;
 
-    /**
-     * 最新章节ID
-     */
     private Long lastChapterId;
 
-    /**
-     * 最新章节名
-     */
     private String lastChapterName;
 
-    /**
-     * 最新章节更新时间
-     */
     private LocalDateTime lastChapterUpdateTime;
 
-    /**
-     * 是否收费;1-收费 0-免费
-     */
-    private Integer isVip;
+    private Byte isVip;
 
-    /**
-     * 创建时间
-     */
     private LocalDateTime createTime;
 
-    /**
-     * 更新时间
-     */
     private LocalDateTime updateTime;
-}
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(Long id) {
+        this.id = id;
+    }
+
+    public Byte getWorkDirection() {
+        return workDirection;
+    }
+
+    public void setWorkDirection(Byte workDirection) {
+        this.workDirection = workDirection;
+    }
+
+    public Long getCategoryId() {
+        return categoryId;
+    }
+
+    public void setCategoryId(Long categoryId) {
+        this.categoryId = categoryId;
+    }
+
+    public String getCategoryName() {
+        return categoryName;
+    }
+
+    public void setCategoryName(String categoryName) {
+        this.categoryName = categoryName;
+    }
+
+    public String getPicUrl() {
+        return picUrl;
+    }
+
+    public void setPicUrl(String picUrl) {
+        this.picUrl = picUrl;
+    }
+
+    public String getBookName() {
+        return bookName;
+    }
+
+    public void setBookName(String bookName) {
+        this.bookName = bookName;
+    }
+
+    public Long getAuthorId() {
+        return authorId;
+    }
+
+    public void setAuthorId(Long authorId) {
+        this.authorId = authorId;
+    }
+
+    public String getAuthorName() {
+        return authorName;
+    }
+
+    public void setAuthorName(String authorName) {
+        this.authorName = authorName;
+    }
+
+    public String getBookDesc() {
+        return bookDesc;
+    }
+
+    public void setBookDesc(String bookDesc) {
+        this.bookDesc = bookDesc;
+    }
+
+    public Byte getScore() {
+        return score;
+    }
+
+    public void setScore(Byte score) {
+        this.score = score;
+    }
+
+    public Byte getBookStatus() {
+        return bookStatus;
+    }
+
+    public void setBookStatus(Byte bookStatus) {
+        this.bookStatus = bookStatus;
+    }
+
+    public Long getVisitCount() {
+        return visitCount;
+    }
+
+    public void setVisitCount(Long visitCount) {
+        this.visitCount = visitCount;
+    }
+
+    public Integer getWordCount() {
+        return wordCount;
+    }
+
+    public void setWordCount(Integer wordCount) {
+        this.wordCount = wordCount;
+    }
+
+    public Integer getCommentCount() {
+        return commentCount;
+    }
+
+    public void setCommentCount(Integer commentCount) {
+        this.commentCount = commentCount;
+    }
+
+    public Long getLastChapterId() {
+        return lastChapterId;
+    }
+
+    public void setLastChapterId(Long lastChapterId) {
+        this.lastChapterId = lastChapterId;
+    }
+
+    public String getLastChapterName() {
+        return lastChapterName;
+    }
+
+    public void setLastChapterName(String lastChapterName) {
+        this.lastChapterName = lastChapterName;
+    }
+
+    public LocalDateTime getLastChapterUpdateTime() {
+        return lastChapterUpdateTime;
+    }
+
+    public void setLastChapterUpdateTime(LocalDateTime lastChapterUpdateTime) {
+        this.lastChapterUpdateTime = lastChapterUpdateTime;
+    }
+
+    public Byte getIsVip() {
+        return isVip;
+    }
+
+    public void setIsVip(Byte isVip) {
+        this.isVip = isVip;
+    }
+
+    public LocalDateTime getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(LocalDateTime createTime) {
+        this.createTime = createTime;
+    }
+
+    public LocalDateTime getUpdateTime() {
+        return updateTime;
+    }
+
+    public void setUpdateTime(LocalDateTime updateTime) {
+        this.updateTime = updateTime;
+    }
+
+    @Override
+    public String toString() {
+        return "BookInfo{" +
+            "id = " + id +
+            ", workDirection = " + workDirection +
+            ", categoryId = " + categoryId +
+            ", categoryName = " + categoryName +
+            ", picUrl = " + picUrl +
+            ", bookName = " + bookName +
+            ", authorId = " + authorId +
+            ", authorName = " + authorName +
+            ", bookDesc = " + bookDesc +
+            ", score = " + score +
+            ", bookStatus = " + bookStatus +
+            ", visitCount = " + visitCount +
+            ", wordCount = " + wordCount +
+            ", commentCount = " + commentCount +
+            ", lastChapterId = " + lastChapterId +
+            ", lastChapterName = " + lastChapterName +
+            ", lastChapterUpdateTime = " + lastChapterUpdateTime +
+            ", isVip = " + isVip +
+            ", createTime = " + createTime +
+            ", updateTime = " + updateTime +
+        "}";
+    }
+}

novel-demo/src/main/java/com/sf/service/IBookChapterService.java

@@ -0,0 +1,16 @@
+package com.sf.service;
+
+import com.sf.po.BookChapter;
+import com.baomidou.mybatisplus.extension.service.IService;
+
+/**
+ * <p>
+ * 小说章节 服务类
+ * </p>
+ *
+ * @author Qing
+ * @since 2023-12-09
+ */
+public interface IBookChapterService extends IService<BookChapter> {
+
+}

novel-demo/src/main/java/com/sf/service/impl/BookChapterServiceImpl.java

@@ -0,0 +1,20 @@
+package com.sf.service.impl;
+
+import com.sf.po.BookChapter;
+import com.sf.mapper.BookChapterMapper;
+import com.sf.service.IBookChapterService;
+import com.baomidou.mybatisplus.extension.service.impl.ServiceImpl;
+import org.springframework.stereotype.Service;
+
+/**
+ * <p>
+ * 小说章节 服务实现类
+ * </p>
+ *
+ * @author Qing
+ * @since 2023-12-09
+ */
+@Service
+public class BookChapterServiceImpl extends ServiceImpl<BookChapterMapper, BookChapter> implements IBookChapterService {
+
+}

novel-demo/src/main/java/com/sf/util/EpubUtils.java

@@ -5,11 +5,13 @@ import nl.siegmann.epublib.epub.EpubReader;
 import org.jsoup.Jsoup;
 import org.jsoup.nodes.Document;
 import org.jsoup.nodes.Element;
+import org.springframework.data.util.Pair;
 
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
@@ -17,8 +19,9 @@ import java.util.regex.Pattern;
 public class EpubUtils {
 
 
-    public static void main(String[] args) {
-        File file = new File("epub/福尔摩斯探案全集.epub");
+    public static List<Pair<String,String>> getContent(String fileName,String bookName) {
+        List<Pair<String, String>> pairList = new ArrayList<>();
+        File file = new File(fileName);
         try {
             FileInputStream fis = new FileInputStream(file);
             EpubReader reader = new EpubReader();
@@ -29,6 +32,7 @@ public class EpubUtils {
             Spine spine = book.getSpine();
 
             List<SpineReference> spineReferences = spine.getSpineReferences();
+
             spineReferences.forEach(spineReference -> {
                 Resource resource = spineReference.getResource();
                 // 因为使用了lambda表达式 所以{}中是一个单独的方法
@@ -45,22 +49,30 @@ public class EpubUtils {
                     String title = doc.title();
                     Element body = doc.body();
                     String bodyHtml = body.html();
-                    if(body.text().isBlank()){
+                    if (body.text().isBlank()) {
                         return;
                     }
+                    if(title.isBlank()){
+                        title = doc.select("p > font > b").text();
+                    }
+                    if(title.isBlank()){
+                        title = bookName;
+                    }
                     String content = removeEleProp(bodyHtml);
+                    Pair<String, String> pair = Pair.of(title, content);
+                    pairList.add(pair);
                     System.out.println();
                 } catch (IOException e) {
                     throw new RuntimeException(e);
                 }
             });
-            System.out.println();
-
+//            System.out.println();
         } catch (FileNotFoundException e) {
             throw new RuntimeException(e);
         } catch (IOException e) {
             throw new RuntimeException(e);
         }
+        return pairList;
     }
 
 

novel-demo/src/main/java/com/sf/util/GeneUtils.java

@@ -20,7 +20,7 @@ public class GeneUtils {
                     builder.author("Qing") // 设置作者
                             .enableSwagger() // 开启 swagger 模式
                             .fileOverride() // 覆盖已生成文件
-                            .outputDir(path + "src/main/java/"); // 指定输出目录
+                            .outputDir(path + "/src/main/java/"); // 指定输出目录
                 })
                 .dataSourceConfig(builder -> builder.typeConvertHandler((globalConfig, typeRegistry, metaInfo) -> {
                     int typeCode = metaInfo.getJdbcType().TYPE_CODE;
@@ -52,7 +52,7 @@ public class GeneUtils {
                 })
                 .strategyConfig(builder -> {
                     // sys_user  t_user   User  UserMapper  UserService
-                    builder.addInclude("book_content"); // 设置需要生成的表名
+                    builder.addInclude("book_chapter"); // 设置需要生成的表名
 //                            .addTablePrefix("t_", "c_"); // 设置过滤表前缀
                 })
                 .templateEngine(new FreemarkerTemplateEngine()) // 使用Freemarker引擎模板，默认的是Velocity引擎模板

novel-demo/src/main/java/com/sf/util/SpiderUtils.java

@@ -12,8 +12,10 @@ import java.util.List;
 
 public class SpiderUtils {
 
-    public static void main(String[] args) throws Exception{
-        String url = "https://www.qidian.com/book/68223/";
+    public static String url = "https://www.qidian.com/book/68223/";
+
+    public static List<Pair<String, String>> getContent(String url) throws Exception {
+        List<Pair<String, String>> contentList = new ArrayList<>();
         Document document = Jsoup.connect(url).get();
 
         String desc = document.select("p#book-intro-detail").text();
@@ -26,29 +28,33 @@ public class SpiderUtils {
         // 数据结构  text href
 //        List<HtmlVo> list = new ArrayList<>();
 //        Pair<String,String> pair = Pair.of("","");
-        List<Pair<String,String>> pairList = new ArrayList<>();
+        List<Pair<String, String>> pairList = new ArrayList<>();
 
         for (Element element : elements) {
             String text = element.text();
             String linkHref = element.attr("href");
             linkHref = "https:" + linkHref;
-            Pair<String,String> pair = Pair.of(text,linkHref);
+            Pair<String, String> pair = Pair.of(text, linkHref);
             pairList.add(pair);
 //            System.out.println();
         }
         System.out.println();
 
-        List<Pair<String,String>> contentList = new ArrayList<>();
+        int count = 20;
         for (Pair<String, String> pair : pairList) {
+            if(count < 0){
+                break;
+            }
             String text = pair.getFirst();
             String href = pair.getSecond();
             Document subDoc = Jsoup.connect(href).get();
             String content = subDoc.select("main").html();
-            Pair<String,String> contentPair = Pair.of(text,content);
+            Pair<String, String> contentPair = Pair.of(text, content);
             contentList.add(contentPair);
-            Thread.sleep(200);
+//            Thread.sleep(200);
             System.out.println();
+            count--;
         }
-
+        return contentList;
     }
 }

novel-demo/src/main/resources/mapper/BookChapterMapper.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.sf.mapper.BookChapterMapper">
+
+</mapper>

novel-demo/src/test/java/com/sf/DBTests.java

@@ -0,0 +1,114 @@
+package com.sf;
+
+import com.sf.mapper.BookChapterMapper;
+import com.sf.mapper.BookContentMapper;
+import com.sf.mapper.BookInfoMapper;
+import com.sf.po.BookChapter;
+import com.sf.po.BookContent;
+import com.sf.po.BookInfo;
+import com.sf.util.EpubUtils;
+import com.sf.util.SpiderUtils;
+import org.junit.jupiter.api.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.data.util.Pair;
+
+import java.time.LocalDateTime;
+import java.util.List;
+
+@SpringBootTest
+public class DBTests {
+
+    @Autowired
+    private BookInfoMapper bookInfoMapper;
+    @Autowired
+    private BookChapterMapper bookChapterMapper;
+    @Autowired
+    private BookContentMapper bookContentMapper;
+
+    @Test
+    public void test() {
+        // select * from book_content where chapter_id in
+        //(select id from book_chapter where book_id = '1431630596354977796')
+
+        // book_info  -> 获得book_id
+        Pair<String, String> bookPair = Pair.of("长夜难明", """
+                该小说讲述了江阳、朱伟等人为了替侯贵平沉冤昭雪，十年来受到重重阻碍但不断收集案件相关证据，最终设局引发社会关注的故事。作者以弱势儿童群体的生存境遇为切口，用“性侵未成年少女”案件将矛头直指偏远地区法治落后的问题，反映出其对寻求司法公正和维护社会秩序的强烈期盼，折射出一个作家的社会责任感
+                """);
+//        BookInfo bookInfo = toBookInfo(bookPair);
+//        bookInfoMapper.insert(bookInfo);
+//        System.out.println(bookInfo.getId());
+
+        List<Pair<String, String>> pairList = EpubUtils.getContent("epub/长夜难明.epub", "长夜难明");
+//        try {
+//            String url = "https://www.qidian.com/book/68223/";
+//            List<Pair<String, String>> pairAnotherList = SpiderUtils.getContent(url);
+//            System.out.println();
+//        } catch (Exception e) {
+//            throw new RuntimeException(e);
+//        }
+
+        Long bookId = 1431630596354977799L;
+        int allWordCount = 0;
+        long lastChapterId = 0;
+        String lastChapterName = "";
+        for (int i = 0; i < pairList.size(); i++) {
+            Pair<String, String> pair = pairList.get(i);
+            int wordCount = pair.getSecond().length();
+            allWordCount += wordCount;
+            BookChapter bookChapter = toBookChapter(bookId, i, pair.getFirst(), wordCount);
+            bookChapterMapper.insert(bookChapter);
+
+            BookContent bookContent = BookContent.builder()
+                    .chapterId(bookChapter.getId())
+                    .content(pair.getSecond())
+                    .createTime(LocalDateTime.now())
+                    .updateTime(LocalDateTime.now())
+                    .build();
+            bookContentMapper.insert(bookContent);
+            System.out.println();
+
+            lastChapterId = bookChapter.getId();
+            lastChapterName = pair.getFirst();
+        }
+
+        System.out.println();
+    }
+
+
+    public BookInfo toBookInfo(Pair<String, String> pair) {
+        BookInfo bookInfo = new BookInfo();
+        // 设置作家信息
+        bookInfo.setAuthorId(1L);
+        bookInfo.setAuthorName("111");
+        // 设置 出版物
+        bookInfo.setWorkDirection((byte) 2);
+        // 出版频道
+        bookInfo.setCategoryId(8L);
+        bookInfo.setCategoryName("出版频道");
+
+        bookInfo.setBookName(pair.getFirst());
+        bookInfo.setBookDesc(pair.getSecond());
+        bookInfo.setPicUrl("/images/default.gif");
+        bookInfo.setBookStatus((byte) 1);
+        bookInfo.setIsVip((byte) 0);
+        bookInfo.setScore((byte) 0);
+        bookInfo.setCreateTime(LocalDateTime.now());
+        bookInfo.setUpdateTime(LocalDateTime.now());
+        return bookInfo;
+    }
+
+    public BookChapter toBookChapter(Long bookId, Integer chapterNum, String chapterName, Integer wordCount) {
+        BookChapter bc = BookChapter.builder()
+                .bookId(bookId)
+                .chapterNum(chapterNum)
+                .chapterName(chapterName)
+                .wordCount(wordCount)
+                .isVip((byte) 0)
+                .createTime(LocalDateTime.now())
+                .updateTime(LocalDateTime.now())
+                .build();
+        return bc;
+    }
+
+}