Java read bytes from Socket on Linux - linux

I'm trying to send a file from my Windows machine to my Raspberry-Pi 2, and I have a client and a server. The client should be able to send a zip file over the network to my server on my linux machine. I know my client and server work on Windows, as when I run both the client and server on windows and connect using 127.0.0.1 it works perfectly. But when sending it to my Pi, nothing gets sent over the socket. Any suggestion?
Server:
package zipsd;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
import java.net.ServerSocket;
import java.net.Socket;
public class Main {
public static void main(String[] args) throws Exception {
if (args.length < 3)
System.out.println("Usage: zipsd <port> <directory> <password>");
else {
int port = Integer.parseInt(args[0]);
String directory = args[1];
String password = args[2];
System.out.println("zipsd: starting server on port " + port);
System.out.println("zipsd: directory = " + directory);
ServerSocket ss = new ServerSocket(port);
System.out.println("zipsd: listening...");
while (true) {
try {
Socket client = ss.accept();
System.out
.println("zipsd: from " + client.getInetAddress());
InputStream input = client.getInputStream();
BufferedReader in = new BufferedReader(
new InputStreamReader(input));
PrintStream out = new PrintStream(client.getOutputStream());
String pwdAttempt = in.readLine();
if (pwdAttempt != null) {
if (!pwdAttempt.equals(password)) {
out.println("[SERVER] zipsd: invalid password");
} else {
out.println("[SERVER] zipsd: authenticated");
String zipName = in.readLine();
if (zipName != null) {
File zipFile = new File(directory + "/"
+ zipName);
try {
FileOutputStream fos = new FileOutputStream(
zipFile);
BufferedOutputStream bos = new BufferedOutputStream(
fos);
byte[] data = new byte[1024 * 1024 * 50];
int count;
while((count = input.read(data)) > 0)
bos.write(data, 0, count);
for(int i = 0; i < 200; i++) //to see if data gets sent, it just prints 0's :(
System.out.println(data[i]);
System.out.println("Got zip file " + zipName);
bos.flush();
fos.close();
bos.close();
out.close();
in.close();
client.close();
} catch (Exception e) {
out.println("[SERVER] zipsd: error in transfer.");
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
}
}
}
}
Client:
package zipsend;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.Socket;
import java.nio.file.Files;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class Main {
public static class ZipUtils {
public static void zipFolder(final File folder, final File zipFile)
throws IOException {
zipFolder(folder, new FileOutputStream(zipFile));
}
public static void zipFolder(final File folder,
final OutputStream outputStream) throws IOException {
try (ZipOutputStream zipOutputStream = new ZipOutputStream(
outputStream)) {
processFolder(folder, zipOutputStream, folder.getPath()
.length() + 1);
zipOutputStream.flush();
zipOutputStream.finish();
zipOutputStream.close();
}
}
private static void processFolder(final File folder,
final ZipOutputStream zipOutputStream, final int prefixLength)
throws IOException {
for (final File file : folder.listFiles()) {
if (file.isFile()) {
final ZipEntry zipEntry = new ZipEntry(file.getPath()
.substring(prefixLength));
zipOutputStream.putNextEntry(zipEntry);
try (FileInputStream inputStream = new FileInputStream(file)) {
byte[] buf = new byte[(int) file.length() + 1];
int read = 0;
while ((read = inputStream.read(buf)) != -1) {
zipOutputStream.write(buf, 0, read);
}
}
zipOutputStream.flush();
zipOutputStream.closeEntry();
} else if (file.isDirectory()) {
processFolder(file, zipOutputStream, prefixLength);
}
}
}
}
public static void main(String[] args) throws Exception {
if(args.length < 4)
System.out.println("Usage: zipsend <folder> <ip> <port> <password>");
else {
String toZip = args[0];
String ip = args[1];
int port = Integer.parseInt(args[2]);
String pwd = args[3];
File folderToZip = new File(toZip);
if(!folderToZip.exists()) {
System.out.println("[ERROR] invalid folder name");
System.exit(1);
}
System.out.print("[INFO] connecting... ");
Socket s = new Socket(ip, port);
System.out.println("OK.");
System.out.println("[INFO] authenticating... ");
BufferedReader in = new BufferedReader(new InputStreamReader(s.getInputStream()));
PrintStream out = new PrintStream(s.getOutputStream());
out.println(pwd);
System.out.println(in.readLine());
System.out.println();
System.out.print("[INFO] zipping " + toZip + "... ");
File zipFile = new File(System.getProperty("user.dir") + "\\" + folderToZip.getName() + ".zip");
ZipOutputStream zout = new ZipOutputStream(new FileOutputStream(zipFile));
ZipUtils.processFolder(folderToZip, zout, folderToZip.getPath().length() + 1);
zout.close();
System.out.println("OK.");
//Transfer file
out.println(zipFile.getName());
byte[] data = new byte[(int)zipFile.length()];
FileInputStream fis = new FileInputStream(zipFile);
BufferedInputStream bis = new BufferedInputStream(fis);
System.out.println("[INFO] sending zip file... ");
OutputStream os = s.getOutputStream();
int count;
while((count = bis.read(data)) > 0) {
os.write(data, 0, count);
}
os.flush();
os.close();
fis.close();
bis.close();
s.close();
System.out.println("[INFO] done. Sent " + Files.size(zipFile.toPath()) + " bytes.");
zipFile.delete();
}
}
}

InputStream input = client.getInputStream();
BufferedReader in = new BufferedReader(new InputStreamReader(input));
Your problem is here. You can't use multiple inputs on a socket when one or more of them is buffered. The buffered input stream/reader will read-ahead and 'steal' data from the other stream. You need to change your protocol so you can use the same stream for the life of the socket at both ends. For example, use DataInputStream, with readUTF() for the file name and read() for the data: at the sender, use DataOutputStream, with writeUTF() for the filename and write() for the data.

Related

Failed to upload a Document Library Entry File in Liferay DXP 7.4

I'm currently developing a Liferay portlet that can upload some pdfs and png to the Documents and Media Repository in Liferay DXP 7.4.13 u33.
I'm trying to use the static method addFileEntry from the class DLFileEntryLocalServiceUtil. Either in its normal or deprecated mode it's not working.
I have also tried with DLAppLocalServiceUtil.addFileEntry(externalReferenceCode, userId, repositoryId, folderId, sourceFileName,mimeType, urlTitle, description, changeLog, file, expirationDate, reviewDate, serviceContext) but I got the same result.
Im getting the following errors from this request, when I call this method via an MVCActionCommand in my portlet:
ERROR [liferay/monitoring-4][ParallelDestination:59] Unable to process message {destinationName=liferay/monitoring, response=null, responseDestinationName=null,
responseId=null, payload=[{displayName=ConceptUploadUrls, portletId=com_liferay_concept_upload_urls_ConceptUploadUrlsPortlet_INSTANCE_KNuJiTyuo6gx,
requestType=ACTION, {attributes=null, companyId=20097, groupId=20121, description=null, duration=0, name=com_liferay_concept_upload_urls_ConceptUploadUrlsPortlet,
namespace=com.liferay.monitoring.Portlet, requestStatus=null, stopWatch=0:00:10.151, timeout=0, user=20125}}, {referer=http://localhost:8080/client, remoteAddr=127.0.0.1,
requestURL=, statusCode=0, userAgent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36, {attributes=null,
companyId=20097, groupId=20121, description=null, duration=10155, name=/c/portal/layout, namespace=com.liferay.monitoring.Portal, requestStatus=ERROR, stopWatch=0:00:10.155,
timeout=-1, user=20125}}, {referer=http://localhost:8080/client, remoteAddr=127.0.0.1, requestURL=, statusCode=0, userAgent=Mozilla/5.0 (Windows NT 10.0; Win64; x64)
AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36, {attributes=null, companyId=20097, groupId=0, description=null, duration=10157, name=/web/guest/carga-por-url,
namespace=com.liferay.monitoring.Portal, requestStatus=ERROR, stopWatch=0:00:10.157, timeout=-1, user=null}}], values={defaultLocale=en_US, companyId=20097, groupId=0, principalName=20125,
permissionChecker=com.liferay.portal.kernel.util.TransientValue#4ff66308, siteDefaultLocale=en_US, themeDisplayLocale=en_US}}
com.liferay.portal.kernel.messaging.MessageListenerException: java.lang.NullPointerException
Caused by: java.lang.NullPointerException at com.liferay.portal.monitoring.internal.statistics.portlet.PortletStatistics.processDataSample(PortletStatistics.java:112) ~[?:?]
...
ERROR [http-nio-8080-exec-3][VirtualHostFilter:381] null
javax.servlet.ServletException: Servlet execution threw an exception
Caused by: java.lang.NoSuchMethodError: com.liferay.document.library.kernel.service.DLAppService.addFileEntry(Ljava/lang/String;
JJLjava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/io/File;Ljava/util/Date;Ljava/util/Date;Lcom/liferay/portal/kernel/service/ServiceContext;)Lcom/liferay/portal/kernel/repository/model/FileEntry;
at com.liferay.concept.upload.urls.portlet.action.UploadFile.uploadFile(UploadFile.java:193) ~[?:?]
...
The complete program is the following:
package com.liferay.concept.upload.urls.portlet.action;
import com.liferay.concept.upload.urls.constants.ConceptUploadUrlsPortletKeys;
import com.liferay.portal.kernel.exception.PortalException;
import com.liferay.portal.kernel.portlet.bridges.mvc.BaseMVCActionCommand;
import com.liferay.portal.kernel.portlet.bridges.mvc.MVCActionCommand;
import com.liferay.portal.kernel.service.ServiceContext;
import com.liferay.portal.kernel.service.ServiceContextFactory;
import com.liferay.portal.kernel.servlet.SessionErrors;
import com.liferay.portal.kernel.servlet.SessionMessages;
import com.liferay.portal.kernel.theme.ThemeDisplay;
import com.liferay.portal.kernel.upload.UploadPortletRequest;
import com.liferay.portal.kernel.util.FileUtil;
import com.liferay.portal.kernel.util.MimeTypesUtil;
import com.liferay.portal.kernel.util.PortalUtil;
import com.liferay.portal.kernel.util.WebKeys;
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import java.util.Map;
import com.liferay.concept.upload.urls.constants.MVCCommandNames;
import com.liferay.document.library.kernel.model.DLFileEntry;
import com.liferay.document.library.kernel.model.DLFolderConstants;
import com.liferay.document.library.kernel.service.DLFileEntryLocalServiceUtil;
import com.liferay.dynamic.data.mapping.kernel.DDMFormValues;
import javax.portlet.ActionRequest;
import javax.portlet.ActionResponse;
import org.osgi.service.component.annotations.Component;
#Component(immediate = true, property = { "javax.portlet.name=" + ConceptUploadUrlsPortletKeys.CONCEPTUPLOADURLS,
"mvc.command.name=" + MVCCommandNames.UPLOAD_FILE }, service = MVCActionCommand.class)
public class UploadFile extends BaseMVCActionCommand {
#Override
protected void doProcessAction(ActionRequest actionRequest, ActionResponse actionResponse) throws Exception {
try {
UploadPortletRequest uploadRequest = PortalUtil.getUploadPortletRequest(actionRequest);
if (uploadRequest.getSize("myFile") == 0) {
SessionErrors.add(actionRequest, "error");
}
// saving the document in local
String fileName = uploadRequest.getFileName("myFile");
File file = uploadRequest.getFile("myFile");
saveDocument(file, fileName);
// readandDownload
List<String> urls = UploadUrlsFromFile(actionRequest, file);
System.out.println("urls Read: " + urls);
SessionMessages.add(actionRequest, "success");
} catch (FileNotFoundException e) {
System.out.println("EXCEPTION! Input File Not Found.");
e.printStackTrace();
SessionMessages.add(actionRequest, "error");
} catch (NullPointerException e) {
System.out.println("EXCEPTION! An Object is null");
e.printStackTrace();
SessionMessages.add(actionRequest, "error");
} catch (IOException e) {
System.out.println("EXCEPTION! Error Reading The Input File.");
SessionMessages.add(actionRequest, "error");
e.printStackTrace();
} catch (PortalException p) {
System.out.println("EXCEPTION! Portal Exception");
p.printStackTrace();
SessionMessages.add(actionRequest, "error");
}
}
private void saveDocument(File file, String fileName) throws IOException {
File newFile = new File(folder + "_" + fileName);
FileInputStream fis = new FileInputStream(file);
FileOutputStream fos = new FileOutputStream(newFile);
System.out.println("File: " + file);
System.out.println("SavedFile: " + newFile);
byte[] bytes_ = FileUtil.getBytes(file);
int i = fis.read(bytes_);
while (i != -1) {
fos.write(bytes_, 0, i);
i = fis.read(bytes_);
}
fis.close();
fos.close();
long size = newFile.length();
System.out.println("File size kb:" + (float) size / 1024);
System.out.println("File created: " + newFile.getName());
}
private List<String> UploadUrlsFromFile(ActionRequest actionRequest, File inputFile)
throws IOException, PortalException {
BufferedReader br = new BufferedReader(new FileReader(inputFile));
String line;
List<String> urls = new ArrayList<String>();
while ((line = br.readLine()) != null) {
urls.add(line);
}
br.close();
for (String url : urls) {
String fileName = getNameForFileFromUrl(url);
// download file from the web
try {
File file = downloadFile(url, fileName);
DLFileEntry dlFileEntry = uploadFile(actionRequest, file);
} catch (NoSuchMethodError e) {
System.out.println("EXCEPTION! Error with the class method.");
SessionMessages.add(actionRequest, "error");
} catch (PortalException e) {
SessionMessages.add(actionRequest, "error");
throw (new PortalException());
} catch (FileNotFoundException e) {
System.out.println("EXCEPTION! File Not Found.");
SessionMessages.add(actionRequest, "error");
} catch (MalformedURLException e) {
System.out.println("EXCEPTION! Malformed URL.");
SessionMessages.add(actionRequest, "error");
} catch (IOException e) {
System.out.println("EXCEPTION! Error reading the File.");
SessionMessages.add(actionRequest, "error");
}
}
return urls;
}
private String getNameForFileFromUrl(String url) {
String fileName = String.valueOf(url);
int index = fileName.lastIndexOf("/");
if (index != -1) {
fileName = fileName.substring(index + 1);
}
return fileName;
}
private File downloadFile(String url, String fileName) throws MalformedURLException, IOException, FileNotFoundException {
String completePath = folder + fileName;
BufferedInputStream in = new BufferedInputStream(new URL(url).openStream());
System.out.println("URL: " + url);
FileOutputStream fileOutputStream = new FileOutputStream(completePath);
byte dataBuffer[] = new byte[1024];
int bytesRead;
while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) {
fileOutputStream.write(dataBuffer, 0, bytesRead);
}
fileOutputStream.close();
File file = new File(completePath);
System.out.println("New File Downloaded: " + completePath);
return file;
}
#SuppressWarnings("deprecation")
private DLFileEntry uploadFile(ActionRequest actionRequest, File file) throws PortalException {
ThemeDisplay themeDisplay = (ThemeDisplay) actionRequest.getAttribute(WebKeys.THEME_DISPLAY);
String fileName = file.getName();
String externalReferenceCode = "";
long userId = themeDisplay.getUserId();
long groupId = themeDisplay.getScopeGroupId();
long repositoryId = groupId;
long folderId = DLFolderConstants.DEFAULT_PARENT_FOLDER_ID;
String sourceFileName = fileName;
String mimeType = MimeTypesUtil.getContentType(file);
String title = FileUtil.stripExtension(fileName);
String urlTitle = FileUtil.stripExtension(fileName).substring(0, 10);
String description = fileName;
String changeLog = "initialization, web source";
long fileEntryType = 0;
Map<String, DDMFormValues> ddmFormValuesMap = null;
InputStream inputStream = null;
long size = file.length();
// Calendar.set
Calendar myCal = Calendar.getInstance();
myCal.set(Calendar.YEAR, 2023);
myCal.set(Calendar.MONTH, 3);
myCal.set(Calendar.DAY_OF_MONTH, 2);
// Date expirationDate = myCal.getTime(); // never expire
myCal.set(Calendar.DAY_OF_MONTH, 1);
// Date reviewDate = myCal.getTime(); // never review
ServiceContext serviceContext = ServiceContextFactory.getInstance(actionRequest); // permissions and assets
System.out.println("size: " + size);
System.out.println("folderId: " + folderId);
System.out.println("Creating file in DL....");
DLFileEntry fileEntry = null;
try {
fileEntry = DLFileEntryLocalServiceUtil.addFileEntry(userId, groupId, repositoryId, folderId, fileName,
mimeType, fileName, "", "", 0, null, file, null, size, serviceContext);
System.out.println("file entry added");
} catch (Exception e) {
System.out.println("Exception File Entry");
e.printStackTrace();
throw (new PortalException());
}
return fileEntry;
}
private String folder = "D:\\LIFERAY-gradebook\\gradebook-workspace\\modules\\concept-upload-urls\\files\\";
}

How to read a compressed (gzip) file without extension in Spark

I am new to Spark and have a fun task in hand where I have to read a bunch of files from S3, which have some xml content in them.
These files are compressed (Gzip) but do not have that extension.
I read some questions on this here where people suggest to extend the default codec in Spark and force a different extension.
But in my case, there is no extension and the files are named in some 16 digit UUID format such as 2c7358ca472ad91057da84adfba.
You can use newAPIHadoopFile (instead of textFile) with a custom/modified TextInputFormat which forces the use of the GzipCodec.
Instead of calling sparkContext.textFile,
// gzip compressed but no .gz extension:
sparkContext.textFile("s3://mybucket/uuid")
we can use the underlying sparkContext.newAPIHadoopFile which allows us to specify how to read the input:
import org.apache.hadoop.mapreduce.lib.input.GzipInputFormatWithoutExtention
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.io.{LongWritable, Text}
sparkContext
.newAPIHadoopFile(
"s3://mybucket/uuid",
classOf[GzipInputFormatWithoutExtention], // This is our custom reader
classOf[LongWritable],
classOf[Text],
new Configuration(sparkContext.hadoopConfiguration)
)
.map { case (_, text) => text.toString }
The usual way of calling newAPIHadoopFile would be with TextInputFormat. This is the part which wraps how the file is read and where the compression codec is chosen based on the file extension.
Let's call it GzipInputFormatWithoutExtention and implement it as follow as an extension of TextInputFormat (this is a Java file and let's put it in package src/main/java/org/apache/hadoop/mapreduce/lib/input):
package org.apache.hadoop.mapreduce.lib.input;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.JobContext;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import com.google.common.base.Charsets;
public class GzipInputFormatWithoutExtention extends TextInputFormat {
public RecordReader<LongWritable, Text> createRecordReader(
InputSplit split,
TaskAttemptContext context
) {
String delimiter =
context.getConfiguration().get("textinputformat.record.delimiter");
byte[] recordDelimiterBytes = null;
if (null != delimiter)
recordDelimiterBytes = delimiter.getBytes(Charsets.UTF_8);
// Here we use our custom `GzipWithoutExtentionLineRecordReader`
// instead of `LineRecordReader`:
return new GzipWithoutExtentionLineRecordReader(recordDelimiterBytes);
}
#Override
protected boolean isSplitable(JobContext context, Path file) {
return false; // gzip isn't a splittable codec (as opposed to bzip2)
}
}
In fact we have to go one level deeper and also replace the default LineRecordReader (Java) with our own (let's call it GzipWithoutExtentionLineRecordReader).
As it's quite difficult to inherit from LineRecordReader, we can copy LineRecordReader (in src/main/java/org/apache/hadoop/mapreduce/lib/input) and slightly modify (and simplify) the initialize(InputSplit genericSplit, TaskAttemptContext context) method by forcing the usage of the Gzip codec:
(the only changes compared to the original LineRecordReader have been given a comment explaining what's happening)
package org.apache.hadoop.mapreduce.lib.input;
import java.io.IOException;
import org.apache.hadoop.io.compress.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.Seekable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.InputSplit;
import org.apache.hadoop.mapreduce.RecordReader;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
#InterfaceAudience.LimitedPrivate({"MapReduce", "Pig"})
#InterfaceStability.Evolving
public class GzipWithoutExtentionLineRecordReader extends RecordReader<LongWritable, Text> {
private static final Logger LOG =
LoggerFactory.getLogger(GzipWithoutExtentionLineRecordReader.class);
public static final String MAX_LINE_LENGTH =
"mapreduce.input.linerecordreader.line.maxlength";
private long start;
private long pos;
private long end;
private SplitLineReader in;
private FSDataInputStream fileIn;
private Seekable filePosition;
private int maxLineLength;
private LongWritable key;
private Text value;
private boolean isCompressedInput;
private Decompressor decompressor;
private byte[] recordDelimiterBytes;
public GzipWithoutExtentionLineRecordReader(byte[] recordDelimiter) {
this.recordDelimiterBytes = recordDelimiter;
}
public void initialize(
InputSplit genericSplit,
TaskAttemptContext context
) throws IOException {
FileSplit split = (FileSplit) genericSplit;
Configuration job = context.getConfiguration();
this.maxLineLength = job.getInt(MAX_LINE_LENGTH, Integer.MAX_VALUE);
start = split.getStart();
end = start + split.getLength();
final Path file = split.getPath();
// open the file and seek to the start of the split
final FileSystem fs = file.getFileSystem(job);
fileIn = fs.open(file);
// This line is modified to force the use of the GzipCodec:
// CompressionCodec codec = new CompressionCodecFactory(job).getCodec(file);
CompressionCodecFactory ccf = new CompressionCodecFactory(job);
CompressionCodec codec = ccf.getCodecByClassName(GzipCodec.class.getName());
// This part has been extremely simplified as we don't have to handle
// all the different codecs:
isCompressedInput = true;
decompressor = CodecPool.getDecompressor(codec);
if (start != 0) {
throw new IOException(
"Cannot seek in " + codec.getClass().getSimpleName() + " compressed stream"
);
}
in = new SplitLineReader(
codec.createInputStream(fileIn, decompressor), job, this.recordDelimiterBytes
);
filePosition = fileIn;
if (start != 0) {
start += in.readLine(new Text(), 0, maxBytesToConsume(start));
}
this.pos = start;
}
private int maxBytesToConsume(long pos) {
return isCompressedInput
? Integer.MAX_VALUE
: (int) Math.max(Math.min(Integer.MAX_VALUE, end - pos), maxLineLength);
}
private long getFilePosition() throws IOException {
long retVal;
if (isCompressedInput && null != filePosition) {
retVal = filePosition.getPos();
} else {
retVal = pos;
}
return retVal;
}
private int skipUtfByteOrderMark() throws IOException {
int newMaxLineLength = (int) Math.min(3L + (long) maxLineLength,
Integer.MAX_VALUE);
int newSize = in.readLine(value, newMaxLineLength, maxBytesToConsume(pos));
pos += newSize;
int textLength = value.getLength();
byte[] textBytes = value.getBytes();
if ((textLength >= 3) && (textBytes[0] == (byte)0xEF) &&
(textBytes[1] == (byte)0xBB) && (textBytes[2] == (byte)0xBF)) {
LOG.info("Found UTF-8 BOM and skipped it");
textLength -= 3;
newSize -= 3;
if (textLength > 0) {
textBytes = value.copyBytes();
value.set(textBytes, 3, textLength);
} else {
value.clear();
}
}
return newSize;
}
public boolean nextKeyValue() throws IOException {
if (key == null) {
key = new LongWritable();
}
key.set(pos);
if (value == null) {
value = new Text();
}
int newSize = 0;
while (getFilePosition() <= end || in.needAdditionalRecordAfterSplit()) {
if (pos == 0) {
newSize = skipUtfByteOrderMark();
} else {
newSize = in.readLine(value, maxLineLength, maxBytesToConsume(pos));
pos += newSize;
}
if ((newSize == 0) || (newSize < maxLineLength)) {
break;
}
LOG.info("Skipped line of size " + newSize + " at pos " +
(pos - newSize));
}
if (newSize == 0) {
key = null;
value = null;
return false;
} else {
return true;
}
}
#Override
public LongWritable getCurrentKey() {
return key;
}
#Override
public Text getCurrentValue() {
return value;
}
public float getProgress() throws IOException {
if (start == end) {
return 0.0f;
} else {
return Math.min(1.0f, (getFilePosition() - start) / (float)(end - start));
}
}
public synchronized void close() throws IOException {
try {
if (in != null) {
in.close();
}
} finally {
if (decompressor != null) {
CodecPool.returnDecompressor(decompressor);
decompressor = null;
}
}
}
}

download XSD with all imports

I use gml (3.1.1) XSDs in XSD for my application. I want to download all gml XSDs in version 3.1.1 in for example zip file. In other words: base xsd is here and I want to download this XSD with all imports in zip file or something like zip file. Is there any application which supports that?
I've found this downloader but it doesn't works for me (I think that this application is not supporting relative paths in imports which occurs in gml.xsd 3.1.1). Any ideas?
QTAssistant's XSR (I am associated with it) has an easy to use function that allows one to automatically import and refactor XSD content as local files from all sorts of sources. In the process it'll update schema location references, etc.
I've made a simple screen capture of the steps involved in achieving a task like this which should demonstrate its usability.
Based on the solution of mschwehl, I made an improved class to achieve the fetch. It suited well with the question. See https://github.com/mfalaize/schema-fetcher
You can achieve this using SOAP UI.
Follow these steps :
Create a project using the WSDL.
Choose your interface and open in interface viewer.
Navigate to the tab 'WSDL Content'.
Use the last icon under the tab 'WSDL Content' : 'Export the entire WSDL and included/imported files to a local directory'.
select the folder where you want the XSDs to be exported to.
Note: SOAPUI will remove all relative paths and will save all XSDs to the same folder.
I have written a simple java-main that does the job and change to relative url's
package dl;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.net.Authenticator;
import java.net.PasswordAuthentication;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Scanner;
import java.util.Set;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathExpression;
import javax.xml.xpath.XPathFactory;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;
public class SchemaPersister {
private static final String EXPORT_FILESYSTEM_ROOT = "C:/export/xsd";
// some caching of the http-responses
private static Map<String,String> _httpContentCache = new HashMap<String,String>();
public static void main(String[] args) {
try {
new SchemaPersister().doIt();
} catch (Exception e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
private void doIt() throws Exception {
// // if you need an inouse-Proxy
// final String authUser = "xxxxx";
// final String authPassword = "xxxx"
//
// System.setProperty("http.proxyHost", "xxxxx");
// System.setProperty("http.proxyPort", "xxxx");
// System.setProperty("http.proxyUser", authUser);
// System.setProperty("http.proxyPassword", authPassword);
//
// Authenticator.setDefault(
// new Authenticator() {
// public PasswordAuthentication getPasswordAuthentication() {
// return new PasswordAuthentication(authUser, authPassword.toCharArray());
// }
// }
// );
//
Set <SchemaElement> allElements = new HashSet<SchemaElement>() ;
// URL url = new URL("file:/C:/xauslaender-nachrichten-administration.xsd");
URL url = new URL("http://www.osci.de/xauslaender141/xauslaender-nachrichten-bamf-abh.xsd");
allElements.add ( new SchemaElement(url));
for (SchemaElement e: allElements) {
System.out.println("processing " + e);
e.doAll();
}
System.out.println("done!");
}
class SchemaElement {
private URL _url;
private String _content;
public List <SchemaElement> _imports ;
public List <SchemaElement> _includes ;
public SchemaElement(URL url) {
this._url = url;
}
public void checkIncludesAndImportsRecursive() throws Exception {
InputStream in = new ByteArrayInputStream(downloadContent() .getBytes("UTF-8"));
DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance();
DocumentBuilder builder = factory.newDocumentBuilder();
Document doc = builder.parse(in);
List<Node> includeNodeList = null;
List<Node> importNodeList = null;
includeNodeList = getXpathAttribute(doc,"/*[local-name()='schema']/*[local-name()='include']");
_includes = new ArrayList <SchemaElement> ();
for ( Node element: includeNodeList) {
Node sl = element.getAttributes().getNamedItem("schemaLocation");
if (sl == null) {
System.out.println(_url + " defines one import but no schemaLocation");
continue;
}
String asStringAttribute = sl.getNodeValue();
URL url = buildUrl(asStringAttribute,_url);
SchemaElement tmp = new SchemaElement(url);
tmp.setSchemaLocation(asStringAttribute);
tmp.checkIncludesAndImportsRecursive();
_includes.add(tmp);
}
importNodeList = getXpathAttribute(doc,"/*[local-name()='schema']/*[local-name()='import']");
_imports = new ArrayList <SchemaElement> ();
for ( Node element: importNodeList) {
Node sl = element.getAttributes().getNamedItem("schemaLocation");
if (sl == null) {
System.out.println(_url + " defines one import but no schemaLocation");
continue;
}
String asStringAttribute = sl.getNodeValue();
URL url = buildUrl(asStringAttribute,_url);
SchemaElement tmp = new SchemaElement(url);
tmp.setSchemaLocation(asStringAttribute);
tmp.checkIncludesAndImportsRecursive();
_imports.add(tmp);
}
in.close();
}
private String schemaLocation;
private void setSchemaLocation(String schemaLocation) {
this.schemaLocation = schemaLocation;
}
// http://stackoverflow.com/questions/10159186/how-to-get-parent-url-in-java
private URL buildUrl(String asStringAttribute, URL parent) throws Exception {
if (asStringAttribute.startsWith("http")) {
return new URL(asStringAttribute);
}
if (asStringAttribute.startsWith("file")) {
return new URL(asStringAttribute);
}
// relative URL
URI parentUri = parent.toURI().getPath().endsWith("/") ? parent.toURI().resolve("..") : parent.toURI().resolve(".");
return new URL(parentUri.toURL().toString() + asStringAttribute );
}
public void doAll() throws Exception {
System.out.println("READ ELEMENTS");
checkIncludesAndImportsRecursive();
System.out.println("PRINTING DEPENDENCYS");
printRecursive(0);
System.out.println("GENERATE OUTPUT");
patchAndPersistRecursive(0);
}
public void patchAndPersistRecursive(int level) throws Exception {
File f = new File(EXPORT_FILESYSTEM_ROOT + File.separator + this.getXDSName() );
System.out.println("FILENAME: " + f.getAbsolutePath());
if (_imports.size() > 0) {
for (int i = 0; i < level; i++) {
System.out.print(" ");
}
System.out.println("IMPORTS");
for (SchemaElement kid : _imports) {
kid.patchAndPersistRecursive(level+1);
}
}
if (_includes.size() > 0) {
for (int i = 0; i < level; i++) {
System.out.print(" ");
}
System.out.println("INCLUDES");
for (SchemaElement kid : _includes) {
kid.patchAndPersistRecursive(level+1);
}
}
String contentTemp = downloadContent();
for (SchemaElement i : _imports ) {
if (i.isHTTP()) {
contentTemp = contentTemp.replace(
"<xs:import schemaLocation=\"" + i.getSchemaLocation() ,
"<xs:import schemaLocation=\"" + i.getXDSName() );
}
}
for (SchemaElement i : _includes ) {
if (i.isHTTP()) {
contentTemp = contentTemp.replace(
"<xs:include schemaLocation=\"" + i.getSchemaLocation(),
"<xs:include schemaLocation=\"" + i.getXDSName() );
}
}
FileOutputStream fos = new FileOutputStream(f);
fos.write(contentTemp.getBytes("UTF-8"));
fos.close();
System.out.println("File written: " + f.getAbsolutePath() );
}
public void printRecursive(int level) {
for (int i = 0; i < level; i++) {
System.out.print(" ");
}
System.out.println(_url.toString());
if (this._imports.size() > 0) {
for (int i = 0; i < level; i++) {
System.out.print(" ");
}
System.out.println("IMPORTS");
for (SchemaElement kid : this._imports) {
kid.printRecursive(level+1);
}
}
if (this._includes.size() > 0) {
for (int i = 0; i < level; i++) {
System.out.print(" ");
}
System.out.println("INCLUDES");
for (SchemaElement kid : this._includes) {
kid.printRecursive(level+1);
}
}
}
String getSchemaLocation() {
return schemaLocation;
}
/**
* removes html:// and replaces / with _
* #return
*/
private String getXDSName() {
String tmp = schemaLocation;
// Root on local File-System -- just grap the last part of it
if (tmp == null) {
tmp = _url.toString().replaceFirst(".*/([^/?]+).*", "$1");
}
if ( isHTTP() ) {
tmp = tmp.replace("http://", "");
tmp = tmp.replace("/", "_");
} else {
tmp = tmp.replace("/", "_");
tmp = tmp.replace("\\", "_");
}
return tmp;
}
private boolean isHTTP() {
return _url.getProtocol().startsWith("http");
}
private String downloadContent() throws Exception {
if (_content == null) {
System.out.println("reading content from " + _url.toString());
if (_httpContentCache.containsKey(_url.toString())) {
this._content = _httpContentCache.get(_url.toString());
System.out.println("Cache hit! " + _url.toString());
} else {
System.out.println("Download " + _url.toString());
Scanner scan = new Scanner(_url.openStream(), "UTF-8");
if (isHTTP()) {
this._content = scan.useDelimiter("\\A").next();
} else {
this._content = scan.useDelimiter("\\Z").next();
}
scan.close();
if (this._content != null) {
_httpContentCache.put(_url.toString(), this._content);
}
}
}
if (_content == null) {
throw new NullPointerException("Content of " + _url.toString() + "is null ");
}
return _content;
}
private List<Node> getXpathAttribute(Document doc, String path) throws Exception {
List <Node> returnList = new ArrayList <Node> ();
XPathFactory xPathfactory = XPathFactory.newInstance();
XPath xpath = xPathfactory.newXPath();
{
XPathExpression expr = xpath.compile(path);
NodeList nodeList = (NodeList) expr.evaluate(doc, XPathConstants.NODESET );
for (int i = 0 ; i < nodeList.getLength(); i++) {
Node n = nodeList.item(i);
returnList.add(n);
}
}
return returnList;
}
#Override
public String toString() {
if (_url != null) {
return _url.toString();
}
return super.toString();
}
}
}
I created a python tool to recursively download XSDs with relative paths in import tags (eg: <import schemaLocation="../../../../abc)
https://github.com/n-a-t-e/xsd_download
After downloading the schema you can use xmllint to validate an XML document
I am using org.apache.xmlbeans.impl.tool.SchemaResourceManager from the xmlbeans project. This class is quick and easy to use.
for example:
SchemaResourceManager manager = new SchemaResourceManager(new File(dir));
manager.process(schemaUris, emptyArray(), false, true, true);
manager.writeCache();
This class has a main method that documents the different options available.

How to Switch to OTA Provisioning option in Wireless toolkit J2ME?

I am new to J2ME and now I have to solve one J2ME project.
Below is my login form I don't know what to do with that but it is giving me the error "javax.microedition.io.ConnectionNotFoundException: TCP open". After searching on the google I got some hint that we have to run the code on "OTA Provisioning option ".
Now I don't know how to do that. I have "Version 2.5.2 for CLDC" of WTK. Can anybody suggest me on that?
package model;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import javax.microedition.rms.RecordEnumeration;
import javax.microedition.rms.RecordStore;
import view.Dialogs;
public class Login {
public static RecordStore rs; // Record store
static final String REC_STORE = "db_Login"; // Name of record store
static RecordEnumeration re;
private String login,password;
private int c;
public static String s1,s2,s3;
public Login()
{
if(LoginSrv.st1.equals("Invalid") && LoginSrv.st2.equals("User"))
{
System.out.println("Im from Invalid User");
new Dialogs();
}
else
{
login = LoginSrv.st1;
password = LoginSrv.st2;
c= LoginSrv.it1;
saveRecord();
}
}
public Login(String log,String pas, String ctr)
{
s1 = log;
s2 = pas;
s3 = ctr;
}
public void saveRecord()
{
try
{
rs = RecordStore.openRecordStore(REC_STORE, true );
re = rs.enumerateRecords(null, null, false);
ByteArrayOutputStream baosdata = new ByteArrayOutputStream();
DataOutputStream daosdata = new DataOutputStream(baosdata);
daosdata.writeUTF(login);
daosdata.writeUTF(password);
daosdata.writeInt(c);
byte[] record = baosdata.toByteArray();
rs.addRecord(record, 0, record.length);
System.out.println("Login record added");
}
catch(Exception e)
{
System.out.println("Im from model Login craete record"+e);
}
}
public static Login getRecord()
{
try
{
rs = RecordStore.openRecordStore(REC_STORE, true );
re = rs.enumerateRecords(null, null, false);
if(re.hasNextElement())
{
byte data[] = rs.getRecord(re.nextRecordId());
ByteArrayInputStream strmBytes = new ByteArrayInputStream(data);
DataInputStream strmDataType = new DataInputStream(strmBytes);
String log = strmDataType.readUTF();
String pass = strmDataType.readUTF();
String counter = strmDataType.readUTF();
return(new Login(log,pass,counter));
}
return null;
}
catch(Exception e)
{
System.out.println("Im from model Login loadRecord"+e);
return null;
}
}
}
-----------------------LoginSrv code ---------------------------------------
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package model;
import com.sun.lwuit.Dialog;
import controller.AppConstants;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
/**
*
* #author sandipp
*/
public class LoginSrv {
private ServCon srv;
public static String st1,st2,log,pas;
public static int it1;
public LoginSrv(String s1, String s2)
{
log = s1;
pas = s2;
it1=0;
try
{
ByteArrayOutputStream baosdata = new ByteArrayOutputStream();
DataOutputStream daosdata = new DataOutputStream(baosdata);
daosdata.writeUTF(s1);
daosdata.writeUTF(s2);
srv = new ServCon(new AppConstants().str1,null,baosdata.toByteArray(),false,false,null);
ByteArrayOutputStream obj = (ByteArrayOutputStream)srv.startTransfer();
byte[] record = obj.toByteArray();
ByteArrayInputStream instr = new ByteArrayInputStream(record);
DataInputStream indat = new DataInputStream(instr);
if(srv.getRc() == 200)
{
String su = indat.readUTF();
if(su.equals("successfull"))
{
st1 =indat.readUTF();
st2 =indat.readUTF();
}
else
{
Dialog.show("Error",su , null,Dialog.TYPE_INFO,null,5000);
}
}
else
{
Dialog.show("Error",srv.getRc() + " " + srv.getRm(), null,Dialog.TYPE_INFO,null,5000);
}
}
catch(Exception e)
{
System.out.println("Im from LoginSrv constructor:"+e);
}
}
}

How can I get log4j to delete old rotating log files?

How can I get log4j to delete old rotating log files? I know I can set up automated jobs (cron for UNIX and scheduled task for Windows), but I want it cross platform, and I want it in our application's log configuration as a part of our application, rather than in separate code outside in OS specific scripting languages. Our application is not written in OS scripting languages, and I don't want to do this part of it in them.
RollingFileAppender does this. You just need to set maxBackupIndex to the highest value for the backup file.
Logs rotate for a reason, so that you only keep so many log files around. In log4j.xml you can add this to your node:
<param name="MaxBackupIndex" value="20"/>
The value tells log4j.xml to only keep 20 rotated log files around. You can limit this to 5 if you want or even 1. If your application isn't logging that much data, and you have 20 log files spanning the last 8 months, but you only need a weeks worth of logs, then I think you need to tweak your log4j.xml "MaxBackupIndex" and "MaxFileSize" params.
Alternatively, if you are using a properties file (instead of the xml) and wish to save 15 files (for example)
log4j.appender.[appenderName].MaxBackupIndex = 15
There is no default value to control deleting old log files created by DailyRollingFileAppender. But you can write your own custom Appender that deletes old log files in much the same way as setting maxBackupIndex does for RollingFileAppender.
Simple instructions found here
From 1:
If you are trying to use the Apache Log4J DailyRollingFileAppender for a daily log file, you may need to want to specify the maximum number of files which should be kept. Just like rolling RollingFileAppender supports maxBackupIndex. But the current version of Log4j (Apache log4j 1.2.16) does not provide any mechanism to delete old log files if you are using DailyRollingFileAppender. I tried to make small modifications in the original version of DailyRollingFileAppender to add maxBackupIndex property. So, it would be possible to clean up old log files which may not be required for future usage.
You can achieve it using custom log4j appender.
MaxNumberOfDays - possibility to set amount of days of rotated log files.
CompressBackups - possibility to archive old logs with zip extension.
package com.example.package;
import org.apache.log4j.FileAppender;
import org.apache.log4j.Layout;
import org.apache.log4j.helpers.LogLog;
import org.apache.log4j.spi.LoggingEvent;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.text.SimpleDateFormat;
import java.util.Calendar;
import java.util.Date;
import java.util.GregorianCalendar;
import java.util.Locale;
import java.util.Optional;
import java.util.TimeZone;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
public class CustomLog4jAppender extends FileAppender {
private static final int TOP_OF_TROUBLE = -1;
private static final int TOP_OF_MINUTE = 0;
private static final int TOP_OF_HOUR = 1;
private static final int HALF_DAY = 2;
private static final int TOP_OF_DAY = 3;
private static final int TOP_OF_WEEK = 4;
private static final int TOP_OF_MONTH = 5;
private String datePattern = "'.'yyyy-MM-dd";
private String compressBackups = "false";
private String maxNumberOfDays = "7";
private String scheduledFilename;
private long nextCheck = System.currentTimeMillis() - 1;
private Date now = new Date();
private SimpleDateFormat sdf;
private RollingCalendar rc = new RollingCalendar();
private static final TimeZone gmtTimeZone = TimeZone.getTimeZone("GMT");
public CustomLog4jAppender() {
}
public CustomLog4jAppender(Layout layout, String filename, String datePattern) throws IOException {
super(layout, filename, true);
this.datePattern = datePattern;
activateOptions();
}
public void setDatePattern(String pattern) {
datePattern = pattern;
}
public String getDatePattern() {
return datePattern;
}
#Override
public void activateOptions() {
super.activateOptions();
if (datePattern != null && fileName != null) {
now.setTime(System.currentTimeMillis());
sdf = new SimpleDateFormat(datePattern);
int type = computeCheckPeriod();
printPeriodicity(type);
rc.setType(type);
File file = new File(fileName);
scheduledFilename = fileName + sdf.format(new Date(file.lastModified()));
} else {
LogLog.error("Either File or DatePattern options are not set for appender [" + name + "].");
}
}
private void printPeriodicity(int type) {
String appender = "Log4J Appender: ";
switch (type) {
case TOP_OF_MINUTE:
LogLog.debug(appender + name + " to be rolled every minute.");
break;
case TOP_OF_HOUR:
LogLog.debug(appender + name + " to be rolled on top of every hour.");
break;
case HALF_DAY:
LogLog.debug(appender + name + " to be rolled at midday and midnight.");
break;
case TOP_OF_DAY:
LogLog.debug(appender + name + " to be rolled at midnight.");
break;
case TOP_OF_WEEK:
LogLog.debug(appender + name + " to be rolled at start of week.");
break;
case TOP_OF_MONTH:
LogLog.debug(appender + name + " to be rolled at start of every month.");
break;
default:
LogLog.warn("Unknown periodicity for appender [" + name + "].");
}
}
private int computeCheckPeriod() {
RollingCalendar rollingCalendar = new RollingCalendar(gmtTimeZone, Locale.ENGLISH);
Date epoch = new Date(0);
if (datePattern != null) {
for (int i = TOP_OF_MINUTE; i <= TOP_OF_MONTH; i++) {
SimpleDateFormat simpleDateFormat = new SimpleDateFormat(datePattern);
simpleDateFormat.setTimeZone(gmtTimeZone);
String r0 = simpleDateFormat.format(epoch);
rollingCalendar.setType(i);
Date next = new Date(rollingCalendar.getNextCheckMillis(epoch));
String r1 = simpleDateFormat.format(next);
if (!r0.equals(r1)) {
return i;
}
}
}
return TOP_OF_TROUBLE;
}
private void rollOver() throws IOException {
if (datePattern == null) {
errorHandler.error("Missing DatePattern option in rollOver().");
return;
}
String datedFilename = fileName + sdf.format(now);
if (scheduledFilename.equals(datedFilename)) {
return;
}
this.closeFile();
File target = new File(scheduledFilename);
if (target.exists()) {
Files.delete(target.toPath());
}
File file = new File(fileName);
boolean result = file.renameTo(target);
if (result) {
LogLog.debug(fileName + " -> " + scheduledFilename);
} else {
LogLog.error("Failed to rename [" + fileName + "] to [" + scheduledFilename + "].");
}
try {
this.setFile(fileName, false, this.bufferedIO, this.bufferSize);
} catch (IOException e) {
errorHandler.error("setFile(" + fileName + ", false) call failed.");
}
scheduledFilename = datedFilename;
}
#Override
protected void subAppend(LoggingEvent event) {
long n = System.currentTimeMillis();
if (n >= nextCheck) {
now.setTime(n);
nextCheck = rc.getNextCheckMillis(now);
try {
cleanupAndRollOver();
} catch (IOException ioe) {
LogLog.error("cleanupAndRollover() failed.", ioe);
}
}
super.subAppend(event);
}
public String getCompressBackups() {
return compressBackups;
}
public void setCompressBackups(String compressBackups) {
this.compressBackups = compressBackups;
}
public String getMaxNumberOfDays() {
return maxNumberOfDays;
}
public void setMaxNumberOfDays(String maxNumberOfDays) {
this.maxNumberOfDays = maxNumberOfDays;
}
protected void cleanupAndRollOver() throws IOException {
File file = new File(fileName);
Calendar cal = Calendar.getInstance();
int maxDays = 7;
try {
maxDays = Integer.parseInt(getMaxNumberOfDays());
} catch (Exception e) {
// just leave it at 7.
}
cal.add(Calendar.DATE, -maxDays);
Date cutoffDate = cal.getTime();
if (file.getParentFile().exists()) {
File[] files = file.getParentFile().listFiles(new StartsWithFileFilter(file.getName(), false));
int nameLength = file.getName().length();
for (File value : Optional.ofNullable(files).orElse(new File[0])) {
String datePart;
try {
datePart = value.getName().substring(nameLength);
Date date = sdf.parse(datePart);
if (date.before(cutoffDate)) {
Files.delete(value.toPath());
} else if (getCompressBackups().equalsIgnoreCase("YES") || getCompressBackups().equalsIgnoreCase("TRUE")) {
zipAndDelete(value);
}
} catch (Exception pe) {
// This isn't a file we should touch (it isn't named correctly)
}
}
}
rollOver();
}
private void zipAndDelete(File file) throws IOException {
if (!file.getName().endsWith(".zip")) {
File zipFile = new File(file.getParent(), file.getName() + ".zip");
try (FileInputStream fis = new FileInputStream(file);
FileOutputStream fos = new FileOutputStream(zipFile);
ZipOutputStream zos = new ZipOutputStream(fos)) {
ZipEntry zipEntry = new ZipEntry(file.getName());
zos.putNextEntry(zipEntry);
byte[] buffer = new byte[4096];
while (true) {
int bytesRead = fis.read(buffer);
if (bytesRead == -1) {
break;
} else {
zos.write(buffer, 0, bytesRead);
}
}
zos.closeEntry();
}
Files.delete(file.toPath());
}
}
class StartsWithFileFilter implements FileFilter {
private String startsWith;
private boolean inclDirs;
StartsWithFileFilter(String startsWith, boolean includeDirectories) {
super();
this.startsWith = startsWith.toUpperCase();
inclDirs = includeDirectories;
}
public boolean accept(File pathname) {
if (!inclDirs && pathname.isDirectory()) {
return false;
} else {
return pathname.getName().toUpperCase().startsWith(startsWith);
}
}
}
class RollingCalendar extends GregorianCalendar {
private static final long serialVersionUID = -3560331770601814177L;
int type = CustomLog4jAppender.TOP_OF_TROUBLE;
RollingCalendar() {
super();
}
RollingCalendar(TimeZone tz, Locale locale) {
super(tz, locale);
}
void setType(int type) {
this.type = type;
}
long getNextCheckMillis(Date now) {
return getNextCheckDate(now).getTime();
}
Date getNextCheckDate(Date now) {
this.setTime(now);
switch (type) {
case CustomLog4jAppender.TOP_OF_MINUTE:
this.set(Calendar.SECOND, 0);
this.set(Calendar.MILLISECOND, 0);
this.add(Calendar.MINUTE, 1);
break;
case CustomLog4jAppender.TOP_OF_HOUR:
this.set(Calendar.MINUTE, 0);
this.set(Calendar.SECOND, 0);
this.set(Calendar.MILLISECOND, 0);
this.add(Calendar.HOUR_OF_DAY, 1);
break;
case CustomLog4jAppender.HALF_DAY:
this.set(Calendar.MINUTE, 0);
this.set(Calendar.SECOND, 0);
this.set(Calendar.MILLISECOND, 0);
int hour = get(Calendar.HOUR_OF_DAY);
if (hour < 12) {
this.set(Calendar.HOUR_OF_DAY, 12);
} else {
this.set(Calendar.HOUR_OF_DAY, 0);
this.add(Calendar.DAY_OF_MONTH, 1);
}
break;
case CustomLog4jAppender.TOP_OF_DAY:
this.set(Calendar.HOUR_OF_DAY, 0);
this.set(Calendar.MINUTE, 0);
this.set(Calendar.SECOND, 0);
this.set(Calendar.MILLISECOND, 0);
this.add(Calendar.DATE, 1);
break;
case CustomLog4jAppender.TOP_OF_WEEK:
this.set(Calendar.DAY_OF_WEEK, getFirstDayOfWeek());
this.set(Calendar.HOUR_OF_DAY, 0);
this.set(Calendar.MINUTE, 0);
this.set(Calendar.SECOND, 0);
this.set(Calendar.MILLISECOND, 0);
this.add(Calendar.WEEK_OF_YEAR, 1);
break;
case CustomLog4jAppender.TOP_OF_MONTH:
this.set(Calendar.DATE, 1);
this.set(Calendar.HOUR_OF_DAY, 0);
this.set(Calendar.MINUTE, 0);
this.set(Calendar.SECOND, 0);
this.set(Calendar.MILLISECOND, 0);
this.add(Calendar.MONTH, 1);
break;
default:
throw new IllegalStateException("Unknown periodicity type.");
}
return getTime();
}
}
}
And use this properties in your log4j config file:
log4j.appender.[appenderName]=com.example.package.CustomLog4jAppender
log4j.appender.[appenderName].File=/logs/app-daily.log
log4j.appender.[appenderName].Append=true
log4j.appender.[appenderName].encoding=UTF-8
log4j.appender.[appenderName].layout=org.apache.log4j.EnhancedPatternLayout
log4j.appender.[appenderName].layout.ConversionPattern=%-5.5p %d %C{1.} - %m%n
log4j.appender.[appenderName].DatePattern='.'yyyy-MM-dd
log4j.appender.[appenderName].MaxNumberOfDays=7
log4j.appender.[appenderName].CompressBackups=true

Resources