Submit11Service.java
/*
* Copyright (C) 2017-2025 Gerd Neugebauer
*
* This file is distributed under the 3-clause BSD license.
* See file LICENSE for details.
*/
package org.ctan.site.services.upload;
import static org.ctan.site.services.util.NullCheck.copyNonNull;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.time.LocalDateTime;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import org.ctan.site.CtanConfiguration;
import org.ctan.site.CtanConfiguration.UploadConfig;
import org.ctan.site.domain.account.User;
import org.ctan.site.domain.catalogue.AuthorRef;
import org.ctan.site.domain.catalogue.Pkg;
import org.ctan.site.domain.catalogue.Upload;
import org.ctan.site.services.DateUtils;
import org.ctan.site.services.mail.MailException;
import org.ctan.site.services.mail.MailService;
import org.ctan.site.services.mail.MailService.Mail;
import org.ctan.site.services.upload.util.AbstractSubmitValidator;
import org.ctan.site.services.upload.util.Messages;
import org.ctan.site.services.util.NullCheck;
import org.ctan.site.stores.LicenseStore;
import org.ctan.site.stores.PkgStore;
import org.ctan.site.stores.TopicStore;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import lombok.NonNull;
import lombok.extern.slf4j.Slf4j;
/**
* This class contains the submit service in version 1.1.
*
* @author <a href="mailto:gene@ctan.org">Gerd Neugebauer</a>
*/
@Slf4j
public class Submit11Service extends AbstractSubmitValidator
implements
SubmitService {
/**
* The field <code>SIZE_ANNOUNCE</code> contains the maximum size for the
* announce field.
*/
private static final int SIZE_ANNOUNCE = 255;
/**
* The field <code>SIZE_ANNOUNCEMENT</code> contains the maximum size for
* the announcement field.
*/
private static final int SIZE_ANNOUNCEMENT = 8192;
/**
* The field <code>SIZE_AUTHOR</code> contains the maximum size for the
* author field.
*/
private static final int SIZE_AUTHOR = 255;
/**
* The field <code>SIZE_BUGTRACKER</code> contains the maximum size for the
* bugtracker field.
*/
private static final int SIZE_BUGTRACKER = 255;
/**
* The field <code>SIZE_CTAN_PATH</code> contains the maximum size for the
* ctanPath field.
*/
private static final int SIZE_CTAN_PATH = 255;
/**
* The field <code>SIZE_DESCRIPTION</code> contains the maximum size for the
* description field.
*/
private static final int SIZE_DESCRIPTION = 4096;
/**
* The field <code>SIZE_DEVELOPMENT</code> contains the maximum size for the
* development field.
*/
private static final int SIZE_DEVELOPMENT = 255;
/**
* The field <code>SIZE_EMAIL</code> contains the maximum size for the email
* field.
*/
private static final int SIZE_EMAIL = 255;
/**
* The field <code>SIZE_HOME</code> contains the maximum size for the home
* field.
*/
private static final int SIZE_HOME = 255;
/**
* The field <code>SIZE_LICENSE</code> contains the maximum size for the
* license field.
*/
private static final int SIZE_LICENSE = 2048;
/**
* The field <code>SIZE_NOTE</code> contains the maximum size for the note
* field.
*/
private static final int SIZE_NOTE = 2048;
/**
* The field <code>SIZE_PKG</code> contains the maximum size for the pkg
* field.
*/
private static final int SIZE_PKG = 32;
/**
* The field <code>SIZE_REPOSITORY</code> contains the maximum size for the
* repository field.
*/
private static final int SIZE_REPOSITORY = 255;
/**
* The field <code>SIZE_SUMMARY</code> contains the maximum size for the
* summary field.
*/
private static final int SIZE_SUMMARY = 128;
/**
* The field <code>SIZE_SUPPORT</code> contains the maximum size for the
* support field.
*/
private static final int SIZE_SUPPORT = 255;
/**
* The field <code>SIZE_TOPIC</code> contains the maximum size for the topic
* field.
*/
private static final int SIZE_TOPIC = 2048;
/**
* The field <code>SIZE_UPDATE</code> contains the maximum size for the
* update field.
*/
private static final int SIZE_UPDATE = 8;
/**
* The field <code>SIZE_UPLOADER</code> contains the maximum size for the
* uploader field.
*/
private static final int SIZE_UPLOADER = 255;
/**
* The field <code>SIZE_VERSION</code> contains the maximum size for the
* vers field.
*/
private static final int SIZE_VERSION = 32;
/**
* The field <code>FIELDS</code> contains the specification of the fields.
*/
private static final Fields FIELDS = Fields.builder()
.pkg(FieldOptions.builder()
.text("CTAN id of the package")
.nullable(false)
.maxsize(SIZE_PKG)
.build())
.version(FieldOptions.builder()
.text("version of the package")
.nullable(false)
.maxsize(SIZE_VERSION)
.build())
.author(FieldOptions.builder()
.text("name of the author(s)")
.nullable(false)
.maxsize(SIZE_AUTHOR)
.build())
.email(FieldOptions.builder()
.text("email address of the uploader")
.email(true)
.blank(false)
.maxsize(SIZE_EMAIL)
.build())
.uploader(FieldOptions.builder()
.text("name of the uploader")
.blank(false)
.maxsize(SIZE_UPLOADER)
.build())
.ctanPath(FieldOptions.builder()
.text("directory on CTAN")
.nullable(true)
.maxsize(SIZE_CTAN_PATH)
.build())
.license(FieldOptions.builder()
.text("license")
.list(true)
.nullable(true)
.maxsize(SIZE_LICENSE)
.build())
.home(FieldOptions.builder()
.text("URL of the home page")
.list(true)
.url(true)
.nullable(true)
.maxsize(SIZE_HOME)
.build())
.bugtracker(FieldOptions.builder()
.text("URL of the bug tracker")
.list(true)
.url(true)
.nullable(true)
.maxsize(SIZE_BUGTRACKER)
.build())
.support(FieldOptions.builder()
.text("support channel")
.list(true)
.url(true)
.nullable(true)
.maxsize(SIZE_SUPPORT)
.build())
.repository(FieldOptions.builder()
.text("version management")
.list(true)
.url(true)
.nullable(true)
.maxsize(SIZE_REPOSITORY)
.build())
.announce(FieldOptions.builder()
.text("announcements list")
.list(true)
.url(true)
.nullable(true)
.maxsize(SIZE_ANNOUNCE)
.build())
.development(FieldOptions.builder()
.text("developer's channel")
.list(true)
.url(true)
.nullable(true)
.maxsize(SIZE_DEVELOPMENT)
.build())
.update(FieldOptions.builder()
.text("update indicator; true for update")
.nullable(true)
.maxsize(SIZE_UPDATE)
.build())
.topic(FieldOptions.builder()
.text("topic ids")
.list(true)
.nullable(true)
.maxsize(SIZE_TOPIC)
.build())
.announcement(FieldOptions.builder()
.text("text for the mail announcement")
.nullable(true)
.maxsize(SIZE_ANNOUNCEMENT)
.build())
.summary(FieldOptions.builder()
.text("one-liner for the package")
.nullable(false)
.maxsize(SIZE_SUMMARY)
.build())
.description(FieldOptions.builder()
.text("descriptive abstract for the package")
.nullable(false)
.maxsize(SIZE_DESCRIPTION)
.build())
.note(FieldOptions.builder()
.text("note to the CTAN upload managers")
.nullable(true)
.maxsize(SIZE_NOTE)
.build())
.file(FieldOptions.builder()
.text("archive file")
.file(true)
.build())
.build();
/**
* The field <code>config</code> contains the configuration.
*/
private CtanConfiguration config;
/**
* The field <code>mailService</code> contains the mail service.
*/
private MailService mailService;
/**
* The field <code>pkgStore</code> contains the package store.
*/
private PkgStore pkgStore;
/**
* The field <code>uploadService</code> contains the upload service.
*/
private UploadService uploadService;
/**
* The field <code>uploadConfig</code> contains the upload configuration.
*/
private UploadConfig uploadConfig;
/**
* This is the constructor for <code>Submit11Service</code>.
*
* @param config the configuration
* @param uploadService the upload service
* @param mailService the mail service
* @param pkgStore the package store
* @param topicStore the topic store
* @param licenseStore the license store
*/
@SuppressFBWarnings(value = {"CT_CONSTRUCTOR_THROW", "EI_EXPOSE_REP2"})
public Submit11Service(@NonNull CtanConfiguration config,
@NonNull UploadService uploadService,
@NonNull MailService mailService,
@NonNull PkgStore pkgStore,
@NonNull TopicStore topicStore,
@NonNull LicenseStore licenseStore) {
super(topicStore, licenseStore);
this.config = config;
this.uploadService = uploadService;
this.pkgStore = pkgStore;
this.mailService = mailService;
this.uploadConfig = config.getUpload();
NullCheck.isNotNullObject(uploadConfig, "config.upload");
NullCheck.isNotNullObject(uploadConfig.getIncoming(),
"config.upload.incoming");
}
/**
* {@inheritDoc}
*
* @see org.ctan.site.services.upload.SubmitService#getFields(java.lang.String)
*/
@Override
@SuppressFBWarnings(value = "EI_EXPOSE_REP")
public Fields getFields(String api) {
return FIELDS;
}
/**
* This is the getter for the version number.
*
* @return the version number
*/
@Override
public String getVersion() {
return "1.1";
}
/**
* The method <code>joinAuthors</code> provides means to concat the list of
* authors into a single string. The separator is semicolon followed by a
* space.
*
* @param authors the authors
* @return the concatenated String
*/
private String joinAuthors(List<AuthorRef> authors) {
return (authors == null
? ""
: authors.stream()
.map(it -> it.getAuthor().toString())
.collect(Collectors.joining("; ")));
}
/**
* The method <code>saveToIncoming</code> provides means to write the
* received data to two files in the incoming directory.
*
* @param upload the upload record to be saved
* @param filename the file name
* @param messages the messages from the validation
* @param user the user or <code>null</code>
* @param bytes the uploaded archive file contents
* @throws IOException in case of an I/O error
*/
@SuppressFBWarnings(value = "DCN_NULLPOINTER_EXCEPTION")
private void saveToIncoming(Upload upload, String filename,
Messages messages, User user, byte[] bytes)
throws IOException {
String incoming = uploadConfig.getIncoming();
log.debug("\n+++ Trying to save to " + incoming);
String key = upload.getPkg();
File incomingDir = new File(incoming);
if (!incomingDir.canWrite()) {
log.error(
"Incoming directory `" + incomingDir + "´ is not writable");
messages.error(
"Technical problem encountered. Please contact the webmaster");
throw new IOException("Upload could not be saved.");
}
key = key.replaceAll("[^-a-zA-Z0-9_.:]", "");
String uploadDirName = DateUtils.formatDate(upload.getDateCreated())
+ "-" + key.toLowerCase();
File uploadDir = new File(incomingDir, uploadDirName);
if (!uploadDir.mkdirs() && !uploadDir.isDirectory()) {
log.error("Upload directory could not be created");
messages.error(
"Technical problem encountered. Please contact the webmaster");
throw new IOException("Upload could not be saved.");
}
try (var out =
new FileOutputStream(new File(uploadDir, filename))) {
out.write(bytes);
// log.info(new String(bytes));
}
File dataFile = new File(uploadDir, filename + ".data");
upload.setServerUrl("file://" + incoming);
uploadService.save(upload);
upload.toFile(dataFile);
String uploadManagers = uploadConfig.getManagers();
if (uploadManagers == null) {
log.error("config.upload.managers is not defined; "
+ "no mail notification sent for " + key);
} else {
var ctanPath = upload.getCtanPath().replaceAll("\\.\\./", "");
var texarchiveDirectory =
new File(config.getTexArchive().getDirectory()
+ '/' + ctanPath);
var ctanPathExists =
ctanPath != null && texarchiveDirectory.exists();
Pkg pkg = pkgStore.getByKey(upload.getPkg());
try {
// var msg =
mailService.send(Mail.builder()
.html(false)
.model(Map.of("upload", NullCheck.copyNonNull(upload),
"messages", messages.toText(),
"pkg", pkg,
"file", uploadDir,
"ctanPathExists", ctanPathExists))
.type("submit-notification")
.to(uploadManagers)
.build());
log.info("Upload `" + key + "´: notification email sent to "
+ uploadManagers);
} catch (NullPointerException
// | MessagingException
| MailException e) {
log.error("Upload `" + key + "´: notification email failed ",
e);
}
}
if (upload.getConfirm()) {
try {
boolean html = user != null && user.getHtmlEmail();
mailService.send(Mail.builder()
.html(html)
.locale("en")
.model(Map.of("upload", copyNonNull(upload),
"messages", html
? messages.toHtml()
: messages.toText()))
.to(upload.getEmail() + " " + upload.getUploader())
.type("submit-confirmation")
.build());
} catch (NullPointerException | MailException e) {
log.error(incoming + ": " + e.toString());
// System.err.println(incoming + ": " + e.toString());
messages.error(
"Technical problem encountered. "
+ "Please contact the webmaster");
}
}
}
/**
* {@inheritDoc}
*
* @see org.ctan.site.services.upload.SubmitService#upload(java.lang.String,
* org.ctan.site.services.upload.SubmitService.UploadData,
* java.lang.String, java.io.InputStream,
* org.ctan.site.domain.account.User)
*/
@Override
public Messages upload(String api, UploadData data, String fileName,
InputStream stream, User user) {
if (api != null && !"1.1".equals(api)) {
return Messages.errorMessage("Invalid API version", api);
}
byte[] bytes;
try {
bytes = stream != null
? stream.readAllBytes()
: new byte[0];
} catch (IOException e) {
log.error("Upload " + data.getPkg() + ": upload failed", e);
return Messages.errorMessage("Upload failed");
}
Messages messages =
validate(api, data, fileName, new ByteArrayInputStream(bytes));
if (messages.hasErrors()) {
return messages.error("Upload failed");
}
String[] licenses = data.getLicenses();
String[] topics = data.getTopics();
Upload upload =
Upload.builder()
.announce(data.getAnnounce())
.announcement(data.getAnnouncement())
.authors(data.getAuthors())
.bugs(data.getBugs())
.confirm(data.isConfirm())
.ctanPath(data.getCtanPath())
.dateCreated(LocalDateTime.now())
.description(data.getDescription())
.development(data.getDevelopment())
.email(data.getEmail())
.home(data.getHome())
.license(licenses == null ? "" : String.join(";", licenses))
.mailinglist(data.getMailinglist())
.note(data.getNotes())
.pkg(data.getPkg())
.repository(data.getRepository())
.status(Upload.TerminationStatus.UPLOAD)
.support(data.getSupport())
.topics(topics == null ? "" : String.join(";", topics))
.type(pkgStore.getByKey(data.getPkg()) == null
? "new"
: data.getAnnouncement() != null
? "announce"
: "silent")
.uploader(data.getUploader())
.vers(data.getVers())
.build();
if (messages.hasErrors()) {
upload.setStatus(Upload.TerminationStatus.UPLOAD_FAILURE);
uploadService.save(upload);
return messages.info("Upload failed");
}
try {
saveToIncoming(upload, fileName, messages, user, bytes);
} catch (IOException e) {
log.error(e.toString());
upload.setStatus(Upload.TerminationStatus.UPLOAD_FAILURE);
uploadService.save(upload);
return messages.error("Upload failed");
}
upload.setStatus(Upload.TerminationStatus.UPLOAD_OK);
uploadService.save(upload);
return messages;
}
/**
* {@inheritDoc}
*
* @see org.ctan.site.services.upload.SubmitService#validate(java.lang.String,
* org.ctan.site.services.upload.SubmitService.UploadData, String,
* InputStream)
*/
@Override
public Messages validate(String api, UploadData data, String filename,
InputStream stream) {
if (api != null && !"1.1".equals(api)) {
return Messages.errorMessage("Invalid API version", api);
}
if (data == null) {
data = new UploadData();
}
Messages messages = new Messages();
// if(params.degrade=="true"){messages.setErrorMode(false);}
var pkg = hasField(messages, "pkg", data.getPkg(), SIZE_PKG,
p -> {
if (!p.matches("[a-zA-Z][a-zA-Z0-9_-]*")) {
messages.error("Illegal package name", p);
} else if (!p.equals(p.toLowerCase())) {
messages.warning("Package name discouraged", p);
p = p.toLowerCase();
}
return p;
});
String vers =
hasField(messages, "version", data.getVers(), SIZE_VERSION,
v -> v.replaceAll("^\\s+", "")
.replaceAll("\\s+$", "")
.replaceAll("\\s+", " "));
Pkg pkgData = (pkg != null ? pkgStore.getByKey(pkg) : null);
boolean dataIsUpdate = data.isUpdate();
if (dataIsUpdate) {
if (pkgData == null) {
messages.error("Updating non-existent package", pkg);
} else if (vers != null && vers.equals(pkgData.getVers())) {
messages.error("Version already exists", pkg, vers);
}
} else if (pkgData != null) {
messages.error("Package already exists", pkg);
}
if (dataIsUpdate && data.getAuthors() == null) {
data.setAuthors(pkgData == null
? "-\"-"
: joinAuthors(pkgData.getAuthors()));
}
hasListField(messages, "author", data.getAuthors(), SIZE_AUTHOR,
!dataIsUpdate);
hasField(messages, "email", data.getEmail(), SIZE_EMAIL,
email -> {
if (!email.matches(".+@.+")) {
messages.errorOrWarning("Email expected", email);
}
return email;
});
hasField(messages, "uploader", data.getUploader(), SIZE_UPLOADER);
var p = hasField(messages, "ctanPath", data.getCtanPath(),
SIZE_CTAN_PATH, false,
path -> validateCtanPath(messages, path,
config.getTexArchive().getDirectory()));
data.setCtanPath(p);
hasListField(messages, "license", data.getLicenses(),
SIZE_LICENSE, !dataIsUpdate,
license -> validateLicense(messages, license));
hasUrlListField(messages, "home", data.getHome(),
SIZE_HOME);
hasUrlListField(messages, "bugtracker", data.getBugs(),
SIZE_BUGTRACKER);
hasUrlListField(messages, "support", data.getSupport(),
SIZE_SUPPORT);
hasUrlListField(messages, "announce", data.getAnnounce(),
SIZE_ANNOUNCE);
hasUrlListField(messages, "repository", data.getRepository(),
SIZE_REPOSITORY);
hasUrlListField(messages, "development", data.getDevelopment(),
SIZE_DEVELOPMENT);
hasListField(messages, "topic", data.getTopics(), SIZE_TOPIC, false,
t -> validateTopic(messages, t));
hasField(messages, "announcement", data.getAnnouncement(),
SIZE_ANNOUNCEMENT, !dataIsUpdate);
hasField(messages, "summary", data.getCaption(),
SIZE_SUMMARY, !dataIsUpdate);
hasField(messages, "description", data.getDescription(),
SIZE_DESCRIPTION, !dataIsUpdate);
hasField(messages, "note", data.getNotes(), SIZE_NOTE, false,
note -> {
if (note.length() >= 16) {
messages.infoMode();
}
return note;
});
return validateArchive(messages, pkg, filename, stream);
}
}