UploadService.java
/*
* Copyright © 2023-2026 The CTAN Team and individual authors
*
* This file is distributed under the 3-clause BSD license.
* See file LICENSE for details.
*/
package org.ctan.site.services.upload;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.LineNumberReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.time.Duration;
import java.time.LocalDateTime;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.apache.commons.codec.digest.DigestUtils;
import org.ctan.site.CtanConfiguration.UploadConfig;
import org.ctan.site.domain.catalogue.Upload;
import org.ctan.site.stores.UploadStore;
import org.ctan.site.stores.base.GeneralPage;
import org.jsoup.Jsoup;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Getter;
import lombok.NoArgsConstructor;
import lombok.NonNull;
/**
* The class <code>UploadService</code> contains the service to access the
* incoming directory.
*
* @author <a href="mailto:gene@ctan.org">Gerd Neugebauer</a>
*/
public class UploadService {
/**
* The class <code>IncomingTo</code> contains the transport object for the
* paged list of incoming files.
*/
@Getter
@AllArgsConstructor
@NoArgsConstructor
@Builder
@SuppressFBWarnings(value = "EI_EXPOSE_REP")
public static class IncomingTo {
/**
* The field <code>page</code> contains the current page. It is 0-based.
*/
private long page;
/**
* The field <code>size</code> contains the page size.
*/
private long size;
/**
* The field <code>length</code> contains the total number of files.
*/
private long length;
/**
* The field <code>files</code> contains the page list of file.
*/
private List<String> files;
}
/**
* The field <code>statDate</code> contains the date of the most recent stat
* call.
*/
private LocalDateTime statDate = LocalDateTime.MIN;
/**
* The field <code>statData</code> contains the cached data.
*/
private Map<String, Integer> statData = null;
/**
* The field <code>base</code> contains the incoming directory.
*/
private String incoming = null;
/**
* The field <code>addendum</code> contains the location of the addendum
* from the configuration.
*/
private String addendum = null;
/**
* The field <code>uploadStore</code> contains the underlying store.
*/
private @NonNull UploadStore uploadStore;
/**
* This is the constructor for the class <code>UploadService</code>.
*
* @param config the CTAN configuration
* @param uploadStore the underlying store
*/
@SuppressFBWarnings(value = {"CT_CONSTRUCTOR_THROW", "EI_EXPOSE_REP2"})
public UploadService(@NonNull UploadConfig config,
@NonNull UploadStore uploadStore) {
this.uploadStore = uploadStore;
incoming = config.getIncoming();
if (incoming == null) {
throw new IllegalArgumentException(
"Missing configuration upload.directory");
}
if (!incoming.endsWith("/")) {
incoming = incoming + "/";
}
addendum = config.getAddendum();
if (addendum == null) {
throw new IllegalArgumentException(
"Missing configuration upload.addendum");
}
}
/**
* The method <code>getById</code> provides means to retrieve the entity by
* its id.
*
* @param id the id
* @return the entity or {@code null} if none is present
*/
public Upload get(@NonNull Long id) {
return uploadStore.get(id);
}
/**
* The method <code>getAddendum</code> provides means to retrieve the
* addendum file.
*
* @return the addendum
* @throws IOException in case of an I/O error
*/
public Map<String, String> getAddendum() throws IOException {
var doc = Jsoup.parse(new File(addendum));
var body = doc.getElementsByTag("body")
.html()
.replaceAll(
"&(TeX|LaTeX|TeXLaTeX|LaTeXTeX|BibTeX|LaTeX2e|LaTeXe);",
"&$1;");
return Map.of("title", doc.getElementsByTag("title").text(),
"content", body,
"lang", "en");
}
/**
* The method <code>getIncoming</code> provides means to retrieve the list
* of unprocessed packages. The packages are taken from the incoming
* directory in the configuration.
*
* <p>
* The list is paged. Thus several segments can be requested.
* </p>
*
* <p>
* Directories starting with a dot are silently ignored.
* </p>
*
* @param query the query pattern
* @param page the current page
* @param size the page size
* @param orderBy the attribute to order by
* @param asc sort ascending
*
* @return the list of current uploads
* @throws FileNotFoundException in case that the incoming directory does
* not exist
*/
public IncomingTo getIncoming(String query, long page, long size,
String orderBy, Boolean asc)
throws FileNotFoundException {
if (page < 0L || size <= 0L) {
throw new IllegalArgumentException();
}
var dir = new File(incoming);
if (!dir.isDirectory()) {
throw new FileNotFoundException();
}
var list = dir
.listFiles((file, s) -> (new File(file, s).isDirectory()
&& !s.startsWith(".")));
if (list == null) {
throw new FileNotFoundException();
}
var len = list.length;
if (page * size >= list.length) {
list = new File[]{};
}
List<String> files = Arrays.asList(list)
.stream()
.map((File x) -> x.getName())
.sorted((a, b) -> a.compareToIgnoreCase(b))
.skip(size * page)
.limit(size)
.collect(Collectors.toList());
return IncomingTo.builder()
.files(files)
.length(len)
.page(page)
.size(size)
.build();
}
/**
* The method <code>getStatistics</code> provides means to retrieve the
* cached upload statistics. The upload statistics contains the number of
* uploads per month.
*
* <p>
* The statistics are cached and updated every 30 minutes.
* </p>
*
* @return the statistics
*/
@SuppressFBWarnings(value = "EI_EXPOSE_REP")
public synchronized Map<String, Integer> getStatistics() {
var now = LocalDateTime.now();
if (statData == null
|| Duration.between(statDate, now).getSeconds() > 60L * 30L) {
statData = uploadStore.getStatistics();
statDate = now;
}
return statData;
}
/**
* The method <code>incomingData</code> provides means to parse.
*
* @param dir the directory
* @return a key-value map of the incoming data
* @throws IOException in case of an error
*/
public Map<String, String> incomingData(@NonNull String dir)
throws IOException {
String pkg = dir.substring(24);
File f = new File(incoming + "/" + dir + "/" + pkg + ".zip.data");
if (!f.canRead()) {
throw new FileNotFoundException();
}
Map<String, String> map = new HashMap<>();
try (var in = new FileReader(f, StandardCharsets.UTF_8);
var input = new LineNumberReader(in)) {
String line;
while ((line = input.readLine()) != null) {
String[] a = line.split(":\\W*");
if (a.length == 0) {
continue;
}
String val = line.replaceFirst("[^:]*:\\W*", "");
switch (a[0]) {
case "Package id":
case "Version":
case "Authors":
case "Uploader":
case "Uploader email":
case "Upload type":
case "Location":
case "License":
case "Home":
case "Announce":
case "Repository":
case "Support":
case "Bugs":
case "Development":
case "Topics":
map.put(a[0], val);
break;
case "Summary":
case "Note":
StringBuilder sb = new StringBuilder();
sb.append(val);
while ((line = input.readLine()) != null) {
if (line.startsWith("---")) {
break;
}
sb.append("\n");
sb.append(line);
}
map.put(a[0], sb.toString());
break;
default:
// Ignored
}
}
}
return map;
}
/**
* The method <code>list</code> provides means to extract a page of items.
*
* @param term the search term
* @param page the current page
* @param pageSize the page size
* @param orderBy the order
* @param asc the indicator for ascending
* @return the page
*/
public GeneralPage list(String term, int page, int pageSize,
String orderBy, boolean asc) {
return uploadStore.list(term, page, pageSize, orderBy, asc);
}
/**
* The method <code>listIncoming</code> provides means to TODO gene.
*
* @param q the query string
* @param page the current page, 1 based
* @param size the page size
* @param orderBy the order attribute or <code>null</code>
* @param asc the indicator for ascending or descending order
* @return a page with results
* @throws FileNotFoundException in case the incoming directory does not
* name a directory
*/
public GeneralPage listIncoming(String q, int page, int size,
String orderBy,
boolean asc)
throws FileNotFoundException {
if (page < 0L || size <= 0L) {
throw new IllegalArgumentException();
}
var dir = new File(incoming);
if (!dir.isDirectory()) {
throw new FileNotFoundException("Missing incoming: " + incoming);
}
var list = dir
.listFiles((file, s) -> (new File(file, s).isDirectory()
&& !s.startsWith(".")));
if (list == null) {
throw new FileNotFoundException(
"Missing incoming files in " + incoming);
}
if (page * size >= list.length) {
list = new File[]{};
}
String[] query = q == null
? new String[0]
: q.toLowerCase().split("\\W");
List<Map<String, Object>> files = Arrays.asList(list)
.stream()
.filter(
(File it) -> it.isDirectory()
&& matches(it.getName().toLowerCase(), query))
.sorted((a, b) -> a.getName().compareToIgnoreCase(b.getName()))
.skip((long) size * page)
.limit(size)
.map((File it) -> listIncomingToMap(it))
.collect(Collectors.toList());
return GeneralPage.builder()
.list(files)
.size(list.length)
.build();
}
/**
* The method <code>remove</code> provides means to remove an entity by its
* id.
*
* @param id the id
* @return {@code true} iff something has been removed
*/
public boolean remove(@NonNull Long id) {
return uploadStore.remove(id);
}
/**
* The method <code>save</code> provides means to store an entity.
*
* @param upload the entity
* @return the entity
*/
public Upload save(Upload upload) {
return uploadStore.save(upload);
}
/**
* The method <code>listIncomingToMap</code> provides means to collect the
* data for the incoming list.
*
* @param dir the directory in the incoming directory
* @return the map
*/
private Map<String, Object> listIncomingToMap(File dir) {
String name = dir.getName();
String pkg = name.substring(24);
File f = new File(dir, pkg + ".zip");
long size;
try {
size = Files.size(f.toPath());
} catch (IOException e) {
size = 0L;
}
String md5;
try (InputStream is = new FileInputStream(f)) {
md5 = DigestUtils.md5Hex(is);
} catch (IOException e) {
md5 = "";
}
return Map.of("id", (Object) name,
"pkg", pkg,
"date", name.substring(0, 10) + " "
+ name.substring(11, 23).replace('-', ':'),
"size", Long.valueOf(size).toString(),
"md5", md5);
}
/**
* The method <code>matches</code> provides means to match several words as
* substring of a string.
*
* @param s the string to match
* @param query the words to search for
* @return <code>true</code> iff the match has been found
*/
private boolean matches(String s, String[] query) {
for (var it : query) {
if (!s.contains(it)) {
return false;
}
}
return true;
}
}