commit 2227fed46032b4d0fb201b6e629a6873a92247de parent 507057f3d7836f7fc6809fed5819abe57050c319 Author: Wim Dupont <wim@wimdupont.com> Date: Fri, 17 Feb 2023 03:18:32 +0100 simplified guide generation with gitlab api Former-commit-id: 18447ae2fef660c5e9eb4f93524d4f949bf994ee Diffstat:
15 files changed, 202 insertions(+), 137 deletions(-)
diff --git a/src/main/java/com/wimdupont/personalweb/PersonalWebApplication.java b/src/main/java/com/wimdupont/personalweb/PersonalWebApplication.java @@ -5,11 +5,13 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.context.ApplicationContext; import org.springframework.context.annotation.ComponentScan; +import org.springframework.data.jpa.repository.config.EnableJpaAuditing; import org.springframework.scheduling.annotation.EnableScheduling; @SpringBootApplication @EnableScheduling @ComponentScan +@EnableJpaAuditing public class PersonalWebApplication { public static void main(String[] args) { diff --git a/src/main/java/com/wimdupont/personalweb/api/GitlabApi.java b/src/main/java/com/wimdupont/personalweb/api/GitlabApi.java @@ -0,0 +1,38 @@ +package com.wimdupont.personalweb.api; + +import com.wimdupont.personalweb.api.dto.RepositoryFile; +import com.wimdupont.personalweb.api.dto.RepositoryTreeItem; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.springframework.web.reactive.function.client.WebClient; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +@Component +public class GitlabApi { + + + private final WebClient webClient; + private final String repositoryUrl; + + public GitlabApi(@Value("${gitlab.repository.url}") String repositoryUrl, + WebClient webClient) { + this.repositoryUrl = repositoryUrl; + this.webClient = webClient; + } + + public Optional<List<RepositoryTreeItem>> getRepositoryTree() { + return webClient.get().uri(repositoryUrl + "/tree").retrieve() + .bodyToFlux(RepositoryTreeItem.class).timeout(Duration.ofSeconds(5)) + .collectList().blockOptional(); + } + + public Optional<RepositoryFile> getRepositoryFile(String filePath) { + return webClient.get().uri(String.format("%s/files/%s?ref=main", repositoryUrl, filePath)) + .retrieve().bodyToMono(RepositoryFile.class).timeout(Duration.ofSeconds(5)) + .blockOptional(); + } + +} diff --git a/src/main/java/com/wimdupont/personalweb/api/dto/RepositoryFile.java b/src/main/java/com/wimdupont/personalweb/api/dto/RepositoryFile.java @@ -0,0 +1,14 @@ +package com.wimdupont.personalweb.api.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public record RepositoryFile ( + + @JsonProperty(value = "file_path") + String path, + @JsonProperty(value = "content_sha256") + String contentSha256, + @JsonProperty(value = "content") + String contentBase64 +){ +} diff --git a/src/main/java/com/wimdupont/personalweb/api/dto/RepositoryTreeItem.java b/src/main/java/com/wimdupont/personalweb/api/dto/RepositoryTreeItem.java @@ -0,0 +1,4 @@ +package com.wimdupont.personalweb.api.dto; + +public record RepositoryTreeItem(String path, String name) { +} diff --git a/src/main/java/com/wimdupont/personalweb/controller/GuideController.java b/src/main/java/com/wimdupont/personalweb/controller/GuideController.java @@ -1,7 +1,6 @@ package com.wimdupont.personalweb.controller; import com.wimdupont.personalweb.converter.GuideMetaToDtoConverter; -import com.wimdupont.personalweb.model.dao.Guide; import com.wimdupont.personalweb.service.GuideService; import org.springframework.stereotype.Controller; import org.springframework.ui.Model; @@ -35,7 +34,7 @@ public class GuideController { @GetMapping(value = "/{name}") public String getGuide(@PathVariable(value = "name") String name, Model model) { var text = Optional.ofNullable(guideService.findGuideByTitle(name) - .orElseThrow().getText()) + .orElseThrow().getHtmlText()) .orElse("Oops, nothing here."); model.addAttribute("guide", text); model.addAttribute("title", name); diff --git a/src/main/java/com/wimdupont/personalweb/converter/GuideMetaToDtoConverter.java b/src/main/java/com/wimdupont/personalweb/converter/GuideMetaToDtoConverter.java @@ -2,24 +2,15 @@ package com.wimdupont.personalweb.converter; import com.wimdupont.personalweb.model.GuideMeta; import com.wimdupont.personalweb.model.dto.GuideMetaDto; +import com.wimdupont.personalweb.util.Constants; import org.springframework.stereotype.Component; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.util.Optional; - @Component public class GuideMetaToDtoConverter { public GuideMetaDto convertForMetaData(GuideMeta guideMeta) { return GuideMetaDto.Builder.newBuilder() - .title(guideMeta.getTitle()) - .createdDate(Optional.ofNullable(guideMeta.getCreatedDate()) - .map(LocalDateTime::toLocalDate) - .orElse(LocalDate.now())) - .lastModifiedDate(Optional.ofNullable(guideMeta.getLastModifiedDate()) - .map(LocalDateTime::toLocalDate) - .orElse(LocalDate.now())) + .title(guideMeta.getPath().replace(Constants.ADOC_SUFFIX, "")) .build(); } } diff --git a/src/main/java/com/wimdupont/personalweb/model/GuideMeta.java b/src/main/java/com/wimdupont/personalweb/model/GuideMeta.java @@ -1,13 +1,7 @@ package com.wimdupont.personalweb.model; -import java.time.LocalDateTime; - - public interface GuideMeta { - String getTitle(); - - LocalDateTime getCreatedDate(); + String getPath(); - LocalDateTime getLastModifiedDate(); } diff --git a/src/main/java/com/wimdupont/personalweb/model/dao/Guide.java b/src/main/java/com/wimdupont/personalweb/model/dao/Guide.java @@ -1,64 +1,61 @@ package com.wimdupont.personalweb.model.dao; import jakarta.persistence.Entity; -import jakarta.persistence.GeneratedValue; +import jakarta.persistence.EntityListeners; import jakarta.persistence.Id; -import org.hibernate.annotations.GenericGenerator; +import jakarta.persistence.Transient; +import org.springframework.data.annotation.CreatedDate; +import org.springframework.data.annotation.LastModifiedDate; +import org.springframework.data.jpa.domain.support.AuditingEntityListener; import java.time.LocalDateTime; @Entity +@EntityListeners(AuditingEntityListener.class) public class Guide { @Id - @GeneratedValue(generator = "uuid") - @GenericGenerator(name = "uuid", strategy = "uuid") - private String id; - private String title; - private String adocUrl; - private String feedUrl; - private String text; + private String path; + private String contentSha256; + @Transient + private String contentBase64; + private String htmlText; + @CreatedDate private LocalDateTime createdDate; - private LocalDateTime lastModifiedDate; - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } + @LastModifiedDate + private LocalDateTime modifiedDate; - public String getTitle() { - return title; + public String getPath() { + return path; } - public void setTitle(String title) { - this.title = title; + public void setPath(String path) { + this.path = path; } - public String getAdocUrl() { - return adocUrl; + public String getContentSha256() { + return contentSha256; } - public void setAdocUrl(String adocUrl) { - this.adocUrl = adocUrl; + public void setContentSha256(String contentSha256) { + this.contentSha256 = contentSha256; } - public String getFeedUrl() { - return feedUrl; + public String getContentBase64() { + return contentBase64; } - public void setFeedUrl(String feedUrl) { - this.feedUrl = feedUrl; + public void setContentBase64(String contentBase64) { + this.contentBase64 = contentBase64; } - public String getText() { - return text; + public String getHtmlText() { + return htmlText; } - public void setText(String text) { - this.text = text; + public void setHtmlText(String htmlText) { + this.htmlText = htmlText; } public LocalDateTime getCreatedDate() { @@ -69,11 +66,73 @@ public class Guide { this.createdDate = createdDate; } - public LocalDateTime getLastModifiedDate() { - return lastModifiedDate; + public LocalDateTime getModifiedDate() { + return modifiedDate; + } + + public void setModifiedDate(LocalDateTime modifiedDate) { + this.modifiedDate = modifiedDate; + } + + protected Guide() { + } + + private Guide(Builder builder) { + path = builder.path; + contentSha256 = builder.contentSha256; + contentBase64 = builder.contentBase64; + htmlText = builder.htmlText; + createdDate = builder.createdDate; + modifiedDate = builder.modifiedDate; } - public void setLastModifiedDate(LocalDateTime lastModifiedDate) { - this.lastModifiedDate = lastModifiedDate; + public static final class Builder { + private String path; + private String contentSha256; + private String contentBase64; + private String htmlText; + private LocalDateTime createdDate; + private LocalDateTime modifiedDate; + + private Builder() { + } + + public static Builder newBuilder() { + return new Builder(); + } + + public Builder path(String val) { + path = val; + return this; + } + + public Builder contentSha256(String val) { + contentSha256 = val; + return this; + } + + public Builder contentBase64(String val) { + contentBase64 = val; + return this; + } + + public Builder htmlText(String val) { + htmlText = val; + return this; + } + + public Builder createdDate(LocalDateTime val) { + createdDate = val; + return this; + } + + public Builder modifiedDate(LocalDateTime val) { + modifiedDate = val; + return this; + } + + public Guide build() { + return new Guide(this); + } } } diff --git a/src/main/java/com/wimdupont/personalweb/model/dto/GuideMetaDto.java b/src/main/java/com/wimdupont/personalweb/model/dto/GuideMetaDto.java @@ -1,23 +1,15 @@ package com.wimdupont.personalweb.model.dto; -import java.time.LocalDate; - public record GuideMetaDto( - String title, - LocalDate createdDate, - LocalDate lastModifiedDate + String title ) { private GuideMetaDto(Builder builder) { - this(builder.title, - builder.createdDate, - builder.lastModifiedDate); + this(builder.title); } public static final class Builder { private String title; - private LocalDate createdDate; - private LocalDate lastModifiedDate; private Builder() { } @@ -31,16 +23,6 @@ public record GuideMetaDto( return this; } - public Builder createdDate(LocalDate val) { - createdDate = val; - return this; - } - - public Builder lastModifiedDate(LocalDate val) { - lastModifiedDate = val; - return this; - } - public GuideMetaDto build() { return new GuideMetaDto(this); } diff --git a/src/main/java/com/wimdupont/personalweb/repository/GuideRepository.java b/src/main/java/com/wimdupont/personalweb/repository/GuideRepository.java @@ -12,9 +12,9 @@ import java.util.Optional; @Repository public interface GuideRepository extends JpaRepository<Guide, String> { - Optional<Guide> findGuideByTitle(String title); + Optional<Guide> findByPath(String path); //Native to exclude columns - @Query(value = "select title, created_date, last_modified_date from guide order by created_date desc", nativeQuery = true) + @Query(value = "select path from guide order by created_date desc", nativeQuery = true) List<GuideMeta> findAllMetaData(); } diff --git a/src/main/java/com/wimdupont/personalweb/service/GuideService.java b/src/main/java/com/wimdupont/personalweb/service/GuideService.java @@ -1,37 +1,33 @@ package com.wimdupont.personalweb.service; -import com.rometools.rome.feed.synd.SyndEntry; -import com.rometools.rome.io.FeedException; +import com.wimdupont.personalweb.api.GitlabApi; import com.wimdupont.personalweb.model.GuideMeta; import com.wimdupont.personalweb.model.dao.Guide; import com.wimdupont.personalweb.repository.GuideRepository; -import com.wimdupont.personalweb.util.DateUtil; +import com.wimdupont.personalweb.util.Constants; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; -import java.io.IOException; import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; import java.util.List; import java.util.Optional; @Service +@Transactional public class GuideService { private static final Logger LOGGER = LoggerFactory.getLogger(GuideService.class); - private final GuideRepository guideRepository; - private final SyndFeedService syndService; + private final GitlabApi gitlabApi; - public GuideService(GuideRepository guideRepository, - SyndFeedService syndService) { + public GuideService(GuideRepository guideRepository, GitlabApi gitlabApi) { this.guideRepository = guideRepository; - this.syndService = syndService; + this.gitlabApi = gitlabApi; } - public Guide update(Guide guide) { + public Guide save(Guide guide) { return guideRepository.save(guide); } @@ -39,31 +35,33 @@ public class GuideService { return guideRepository.findAllMetaData(); } + public Optional<Guide> findByPath(String path) { + return guideRepository.findByPath(path); + } + + public List<Guide> findAllToGenerate() { var guidesToGenerate = new ArrayList<Guide>(); - for (Guide guide : guideRepository.findAll()) { - try { - var syndFeed = syndService.getSyndFeedForUrl(guide.getFeedUrl()); - var lastPubDate = DateUtil.toLocalDateTime(syndFeed.getPublishedDate()); - if (guide.getText() == null || guide.getLastModifiedDate() == null || lastPubDate.isAfter(guide.getLastModifiedDate())) { - var dateCreated = DateUtil.toLocalDateTime(Collections.min(syndFeed.getEntries(), - Comparator.comparing(SyndEntry::getUpdatedDate)).getUpdatedDate()); - guide.setCreatedDate(dateCreated); - guide.setLastModifiedDate(lastPubDate); - guidesToGenerate.add(guide); - } - } catch (IOException | FeedException e) { - throw new RuntimeException(e); - } - } + gitlabApi.getRepositoryTree().orElseThrow() + .forEach(repositoryTreeItem -> gitlabApi.getRepositoryFile(repositoryTreeItem.path()) + .filter(repositoryFile -> findByPath(repositoryFile.path()) + .map(guide -> guide.getContentSha256().equals(repositoryFile.contentSha256())) + .isEmpty()) + .map(repositoryFile -> Guide.Builder.newBuilder() + .contentSha256(repositoryFile.contentSha256()) + .contentBase64(repositoryFile.contentBase64()) + .path(repositoryFile.path()) + .build()) + .ifPresent(guidesToGenerate::add)); + if (guidesToGenerate.isEmpty()) { - LOGGER.info("All guides are up to date."); + LOGGER.info("No new guides to generate."); } return guidesToGenerate; } public Optional<Guide> findGuideByTitle(String title) { - return guideRepository.findGuideByTitle(title); + return findByPath(title + Constants.ADOC_SUFFIX); } } diff --git a/src/main/java/com/wimdupont/personalweb/service/ScheduledFileGenerator.java b/src/main/java/com/wimdupont/personalweb/service/ScheduledFileGenerator.java @@ -9,11 +9,10 @@ import org.springframework.stereotype.Service; import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.net.URL; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; +import java.util.Base64; import static com.wimdupont.personalweb.util.Constants.ADOC_SUFFIX; @@ -47,16 +46,9 @@ public class ScheduledFileGenerator { private void generateGuides(GuideService guideService) { for (Guide guide : guideService.findAllToGenerate()) { - try { - var url = new URL(guide.getAdocUrl()); - try (InputStream in = url.openStream()) { - var adocFile = new String(in.readAllBytes(), StandardCharsets.UTF_8); - guide.setText(adocConverter.convert(adocFile)); - LOG.info("Guide updated: {}", guideService.update(guide)); - } - } catch (IOException e) { - LOG.error(e.getMessage(), e); - } + guide.setHtmlText(adocConverter.convert(new String(Base64.getDecoder() + .decode(guide.getContentBase64().getBytes(StandardCharsets.UTF_8))))); + LOG.info("Guide saved: {}", guideService.save(guide)); } } diff --git a/src/main/resources/db/migration/V1_1__add-guide-table.sql b/src/main/resources/db/migration/V1_1__add-guide-table.sql @@ -1,9 +1,7 @@ CREATE TABLE IF NOT EXISTS guide ( - id VARCHAR(36) primary key NOT NULL, - adoc_url VARCHAR(400) UNIQUE NOT NULL, - feed_url VARCHAR(400) UNIQUE NOT NULL, - title VARCHAR(100) UNIQUE NOT NULL, - text TEXT, + path VARCHAR(200) primary key NOT NULL, + content_sha256 VARCHAR(256) UNIQUE NOT NULL, + html_text TEXT, created_date DATETIME, - last_modified_date DATETIME + modified_date DATETIME ) diff --git a/src/main/resources/db/migration/V1_2_0__insert-arch-install-guide.sql b/src/main/resources/db/migration/V1_2_0__insert-arch-install-guide.sql @@ -1,7 +0,0 @@ -INSERT INTO guide (id, adoc_url, feed_url, title, text, created_date, last_modified_date) VALUES - (UUID(), - 'https://gitlab.com/WimDupont/guides/-/raw/main/Arch%20Linux%20encrypted%20installation.adoc', - 'https://gitlab.com/WimDupont/guides/-/commits/main/Arch%20Linux%20encrypted%20installation.adoc?feed_token=PnjNF_m7PMKTraxkHzcq&format=atom', - "Arch Linux encrypted installation guide", - null, null, null) -; diff --git a/src/main/resources/templates/guides.html b/src/main/resources/templates/guides.html @@ -10,10 +10,11 @@ <h1>Guides</h1> <div th:insert="~{navigation}"/> </header> - <tr th:each="guide : ${guides}"> - <p><a th:href="@{/guides/{guide}(guide=${guide.title})}">[[${guide.title}]]</a> - - [[${guide.createdDate}]] <i><small>(last updated: [[${guide.lastModifiedDate}]])</small></i></p> - </tr> + <ul> + <tr th:each="guide : ${guides}"> + <li><a th:href="@{/guides/{guide}(guide=${guide.title})}">[[${guide.title}]]</a></li> + </tr> + </ul> </div> </body> </html>