package org.crosswire.repo; import java.io.BufferedReader; import java.io.File; import java.io.FileWriter; import java.nio.channels.FileChannel; import java.nio.MappedByteBuffer; import java.nio.charset.Charset; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Vector; import java.util.Comparator; import java.util.Arrays; import java.util.Date; import java.text.SimpleDateFormat; import org.crosswire.utils.HTTPUtils; import org.crosswire.utils.Utils; import org.apache.log4j.Logger; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.CopyOnWriteArraySet; import java.util.Set; public class VersionedRepo { public static String repoRoot = "/home/liferay/repos/NT/"; public static String repoSubfolder[] = new String[] { "transcriptions/", "projects/" }; public static String gitCmd = "/usr/bin/git"; public static String defaultComment = "from INTF"; public static String defaultCommitter = "ECM"; private static Logger logger = Logger.getLogger(VersionedRepo.class); private static boolean useWorkQueue = false; private static boolean isUseWorkQueue() { return useWorkQueue; } public static synchronized void setUseWorkQueue(boolean val) { useWorkQueue = val; for (WorkQueueWorker w : WorkQueueWorker.alive) { w.terminate(); } if (useWorkQueue) { new WorkQueueWorker().start(); } } private static class Work { String [] runCommandParams = null; Work(String []runCommandParams) { this.runCommandParams = runCommandParams; } }; private static ConcurrentLinkedQueue workQueue = new ConcurrentLinkedQueue(); private static class WorkQueueWorker extends Thread { public static Set alive = new CopyOnWriteArraySet(); private volatile boolean running = true; public void terminate() { running = false; } public void run() { logger.info("VersionedRepo Worker Thread (" + Thread.currentThread().getId() + ") starting with " + workQueue.size() + " jobs in the queue."); while (running) { try { alive.add(this); logger.debug("VersionedRepo Worker Thread (" + Thread.currentThread().getId() + ") of ("+alive.size()+") waking up to " + workQueue.size() + " jobs in the queue."); Work w = workQueue.poll(); if (w != null) { // do the work runCommand(w.runCommandParams, null, null, null); } else Thread.sleep(3000); } catch (Exception e) { logger.error("problem in VersionedRepo work queue.", e); } } alive.remove(this); logger.info("VersionedRepo Worker Thread (" + Thread.currentThread().getId() + ") terminating with " + workQueue.size() + " jobs still in the queue."); } } static { try { String val = Utils.getSysConfig().getProperty("VersionedRepoRootFolder"); if (val != null) repoRoot = val; val = Utils.getSysConfig().getProperty("VersionedRepoDefaultCommitter"); if (val != null) defaultCommitter = val; val = Utils.getSysConfig().getProperty("VersionedRepoDefaultComment"); if (val != null) defaultComment = val; val = Utils.getSysConfig().getProperty("VersionedRepoTranscriptionsSubfolder"); if (val != null) repoSubfolder[0] = val; val = Utils.getSysConfig().getProperty("VersionedRepoProjectsSubfolder"); if (val != null) repoSubfolder[1] = val; val = Utils.getSysConfig().getProperty("VersionedRepoGitCmd"); if (val != null) gitCmd = val; } catch (Exception e) { e.printStackTrace(); } if (!repoRoot.endsWith("/")) repoRoot += "/"; if (!repoSubfolder[0].endsWith("/")) repoSubfolder[0] += "/"; if (!repoSubfolder[1].endsWith("/")) repoSubfolder[1] += "/"; } private static SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss Z"); public static class History { public History() {} public History(String name, String logString) { //System.out.println("History Log String: " + logString); this.name = name; int l = 0; int s = logString.indexOf("|", l); if (s > -1) { versionHash = logString.substring(l, s); l = s+1; s = logString.indexOf("|", l); if (s > -1) { author = logString.substring(l, s); l = s+1; s = logString.indexOf("|", l); if (s > -1) { try { date = df.parse(logString.substring(l, s)); } catch (Exception e) { e.printStackTrace(); } l = s+1; s = logString.indexOf("|", l); if (s > -1) { committer = logString.substring(l, s); l = s+1; comment = logString.substring(l); } } } } } public String name = null; public String versionHash = null; public String author = null; public Date date = null; public String committer = null; public String comment = null; public String toFormattedXML() { StringBuffer retVal = new StringBuffer(" 0) { retVal.append(">"); retVal.append(HTTPUtils.canonize(comment)); retVal.append(""); retVal.append(""); } else retVal.append("/>"); return retVal.toString(); } } public static boolean assurePath(String path) { if (path == null) return false; // assert we have a parent path File p = new File(path); if (p.exists()) return false; assurePath(p.getParent()); try { p.mkdir(); runCommand(new String [] { gitCmd, "add", p.getCanonicalPath() }, null, null, null); return true; } catch (IOException e) { e.printStackTrace(); } return false; } public static int PATH_DEFAULT = 0; public static int PATH_PROJECT = 1; public static synchronized void removeFile(String path, String author) { removeFile(path, author, PATH_DEFAULT); } public static synchronized void removeFile(String path, String author, int subPath) { if (author == null || author.trim().length() == 0) author = defaultCommitter; File file = new File(repoRoot + repoSubfolder[subPath], path); try { String cmd[] = new String [] { gitCmd, "rm", file.getCanonicalPath() }; if (useWorkQueue) workQueue.add(new Work(cmd)); else runCommand(cmd, null, null, null); } catch (IOException e) { e.printStackTrace(); } try { String cmd[] = new String [] { gitCmd, "commit", "-m", defaultComment + ", removed file", "--author="+author+" <"+author+"@ntvmr.uni-muenster.de>", file.getCanonicalPath() }; if (useWorkQueue) workQueue.add(new Work(cmd)); else runCommand(cmd, null, null, null); } catch (Exception e) { e.printStackTrace(); } /* TODO: don't do this failsafe alway because we don't know if the workQueue is done with our job yet. */ if (!useWorkQueue) { if (file.exists()) { file.delete(); } } } public static synchronized void moveFile(String path, String newPath, String author) { moveFile(path, newPath, author, PATH_DEFAULT); } public static synchronized void moveFile(String path, String newPath, String author, int subPath) { if (author == null || author.trim().length() == 0) author = defaultCommitter; File file = new File(repoRoot + repoSubfolder[subPath], path); File newFile = new File(repoRoot + repoSubfolder[subPath], newPath); try { String cmd[] = new String [] { gitCmd, "mv", file.getCanonicalPath(), newFile.getCanonicalPath() }; if (useWorkQueue) workQueue.add(new Work(cmd)); else runCommand(cmd, null, null, null); } catch (IOException e) { e.printStackTrace(); } try { String cmd[] = new String [] { gitCmd, "commit", "-m", defaultComment + ", moved file/folder from: " + path, "--author="+author+" <"+author+"@ntvmr.uni-muenster.de>", newFile.getCanonicalPath() }; if (useWorkQueue) workQueue.add(new Work(cmd)); else runCommand(cmd, null, null, null); } catch (Exception e) { e.printStackTrace(); } /* TODO: don't do this failsafe alway because we don't know if the workQueue is done with our job yet. */ if (!useWorkQueue) { if (file.exists()) { file.renameTo(newFile); } } } public static synchronized void putFile(String path, String author, StringBuffer content) { putFile(path, author, content, PATH_DEFAULT); } public static synchronized void putFile(String path, String author, StringBuffer content, int subPath) { if (author == null || author.trim().length() == 0) author = defaultCommitter; File file = new File(repoRoot + repoSubfolder[subPath], path); assurePath(file.getParent()); if (file.exists()) { file.delete(); } try { FileWriter out = new FileWriter(file); out.write(content.toString()); out.close(); String cmd[] = new String [] { gitCmd, "add", file.getCanonicalPath() }; if (useWorkQueue) workQueue.add(new Work(cmd)); else runCommand(cmd, null, null, null); } catch (IOException e) { e.printStackTrace(); } try { String cmd[] = new String [] { gitCmd, "commit", "-m", defaultComment, "--author="+author+" <"+author+"@ntvmr.uni-muenster.de>", file.getCanonicalPath() }; if (useWorkQueue) workQueue.add(new Work(cmd)); else runCommand(cmd, null, null, null); } catch (Exception e) { e.printStackTrace(); } } public static StringBuffer getFile(String path, String author) { return getFile(path, author, null); } public static StringBuffer getFile(String path, String author, String versionHash) { return getFile(path, author, versionHash, PATH_DEFAULT); } public static StringBuffer getFile(String path, String author, String versionHash, int subPath) { StringBuffer retVal = null; File file = new File(repoRoot + repoSubfolder[subPath], path); if (file.exists()) { retVal = new StringBuffer(); if (versionHash != null) { runCommand(new String [] { gitCmd, "show", versionHash+":"+repoSubfolder[subPath]+path }, retVal, null, null); } else { FileInputStream stream = null; try { stream = new FileInputStream(file); FileChannel fc = stream.getChannel(); MappedByteBuffer bb = fc.map(FileChannel.MapMode.READ_ONLY, 0, fc.size()); /* Instead of using default, pass in a decoder. */ retVal.append(Charset.defaultCharset().decode(bb).toString()); } catch (Exception e) {} finally { try { if (stream != null) stream.close(); } catch (Exception e) {} } } } return retVal; } public static History[] getFileHistory(String path, String author) { return getFileHistory(path, author, PATH_DEFAULT); } public static History[] getFileHistory(String path, String author, int subPath) { Vector retVal = new Vector(); File file = new File(repoRoot + repoSubfolder[subPath], path); if (file.exists()) { StringBuffer historyBuffer = new StringBuffer(); runCommand(new String [] { gitCmd, "log", "--follow", "--remove-empty", "--pretty=format:%H|%an|%ai|%ce|%s", repoSubfolder[subPath]+path }, historyBuffer, null, null); for (String h : historyBuffer.toString().split("\n")) { retVal.add(new History(path, h)); } } return (History []) retVal.toArray(new History[0]); } public static File getFileName(String path) { return getFileName(path, PATH_DEFAULT); } public static File getFileName(String path, int subPath) { File base = new File(repoRoot + repoSubfolder[subPath], path); return (base.exists()) ? base : null; } // search for stuff like "initial/scribe" will find all folders // named 'scribe' with an immediate parent named 'initial' public static File[] search(String path) { return search(path, null); } public static File[] search(String path, File start) { return search(path, start, PATH_DEFAULT); } public static File[] search(String path, File start, int subPath) { String segment[] = path.split("/"); File base = start != null ? start : new File(repoRoot + repoSubfolder[subPath]); File dirs[] = base.listFiles(); // do we really care about sort order? Arrays.sort(dirs, new Comparator() { public int compare(File f1, File f2) { return f1.getName().compareTo(f2.getName()); } }); Vector retVal = new Vector(); for (File f : dirs) { if (f.isDirectory()) { retVal.addAll(Arrays.asList(search(path, f))); } } // see if we can short circut checking file name if we know our parent doesn't match if (segment.length > 1) { File parentCheck = base; for (int i = segment.length - 2; i >= 0; --i) { if ("*".equals(segment[i]) || segment[i].equalsIgnoreCase(parentCheck.getName())) { parentCheck = parentCheck.getParentFile(); } else return retVal.toArray(new File[0]); } } for (File f : dirs) { if (segment[segment.length-1].equalsIgnoreCase(f.getName()) || (segment[segment.length-1].startsWith("*") && f.getName().endsWith(segment[segment.length-1].substring(1)))) { retVal.add(f); } } return retVal.toArray(new File[0]); } public static File[] getFolders(String path) { return getFolders(path, PATH_DEFAULT); } public static File[] getFolders(String path, int subPath) { File base = new File(repoRoot + repoSubfolder[subPath], path); File dirs[] = base.listFiles(); if (dirs == null) dirs = new File[0]; Arrays.sort(dirs, new Comparator() { public int compare(File f1, File f2) { return f1.getName().compareTo(f2.getName()); } }); return dirs; } public static synchronized void push() { try { runCommand(new String [] { gitCmd, "push" }, null, null, null); } catch (Exception e) { e.printStackTrace(); } } public static Thread startSlurpStreamThreaded(final InputStream is, final StringBuffer outBuf) { Thread t = new Thread() { InputStream myIS = is; StringBuffer myOutBuf = outBuf; public void run() { try { InputStreamReader isr = new InputStreamReader(myIS); BufferedReader br = new BufferedReader(isr); String line=null; while ( (line = br.readLine()) != null) myOutBuf.append(line + "\n"); } catch (IOException ioe) { ioe.printStackTrace(); } } }; t.start(); return t; } public static int runCommand(String command[], StringBuffer result, StringBuffer error, String toStdin) { int retVal = -1; try { if (error == null) error = new StringBuffer(); if (result == null) result = new StringBuffer(); File cwd = new File(repoRoot); String cmd = ""; for (String s: command) { cmd += " [" + s + "]"; } logger.debug("executing from cwd ("+cwd.getAbsolutePath()+"):" + cmd); java.lang.Process p = Runtime.getRuntime().exec(command, new String[0], cwd); if (toStdin != null) { p.getOutputStream().write(toStdin.getBytes(Charset.forName("UTF-8"))); } Thread o = startSlurpStreamThreaded(p.getInputStream(), result); Thread e = startSlurpStreamThreaded(p.getErrorStream(), error); p.getOutputStream().close(); retVal = p.waitFor(); o.join(); e.join(); if (error.length() > 0) { logger.warn("error: " + error.toString()); } } catch (Exception e) { logger.error("exception executing command: " + error.toString(), e); } logger.debug("returned: " + retVal + "; stdout: " + result); return retVal; } }