Yi EungJun 2014-01-27
code: Limit diff rendering to avoid server overload
* If diff of a file exceeds 500 KB and/or 5,000 lines, do not show the
  diff.
* If a diff exceeds 500 KB, 15,000 lines and/or 2,000 files, do not
  show the overflow.

Note: These limits are choosen roughly. Feel free to modify them if you
find better ones.
@b9ae7bba46b99c9a51922b4433cefbdeb34cf7b5
app/playRepository/FileDiff.java
--- app/playRepository/FileDiff.java
+++ app/playRepository/FileDiff.java
@@ -7,9 +7,7 @@
 import org.eclipse.jgit.diff.RawText;
 import org.eclipse.jgit.lib.FileMode;
 
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
+import java.util.*;
 
 
 /**
@@ -17,6 +15,8 @@
  * https://github.com/eclipse/jgit/blob/v2.3.1.201302201838-r/org.eclipse.jgit/src/org/eclipse/jgit/diff/DiffFormatter.java
  */
 public class FileDiff {
+    public static final int SIZE_LIMIT = 500 * 1024;
+    public static final int LINE_LIMIT = 5000;
     public RawText a;
     public RawText b;
     public EditList editList;
@@ -32,15 +32,46 @@
     public CodeComment.Side interestSide = null;
     public FileMode oldMode;
     public FileMode newMode;
+    private Hunks hunks;
+
+    public static class Hunks extends ArrayList<Hunk> {
+        private static final long serialVersionUID = -2359650678446017697L;
+        public int size;
+        public int lines;
+    }
+
+    public static class SizeExceededHunks extends Hunks {
+        private static final long serialVersionUID = 3089104397758709369L;
+    }
+
+    public static boolean isRawTextSizeExceeds(RawText rawText) {
+        return getRawTextSize(rawText) > SIZE_LIMIT || rawText.size() > LINE_LIMIT;
+    }
+
+    public static int getRawTextSize(RawText rawText) {
+        int size = 0;
+        for(int i = 0; i < rawText.size(); i++) {
+            size += rawText.getString(i).length();
+        }
+        return size;
+    }
 
     /**
      * Get list of hunks
-     *
-     * @throws java.io.IOException
      */
-    public List<Hunk> getHunks() {
+    public Hunks getHunks() {
+        if (hunks != null) {
+            return hunks;
+        }
 
-        List<Hunk> hunks = new ArrayList<>();
+        if (editList == null) {
+            return null;
+        }
+
+        int size = 0;
+        int lines = 0;
+
+        hunks = new Hunks();
 
         for (int curIdx = 0; curIdx < editList.size();) {
             Hunk hunk = new Hunk();
@@ -84,12 +115,16 @@
                     case A:
                         if (hunk.beginA <= interestLine && hunk.endA >= interestLine) {
                             hunks.add(hunk);
+                            size += hunk.size();
+                            lines += hunk.lines.size();
                             added = true;
                         }
                         break;
                     case B:
                         if (hunk.beginB <= interestLine && hunk.endB >= interestLine) {
                             hunks.add(hunk);
+                            size += hunk.size();
+                            lines += hunk.lines.size();
                             added = true;
                         }
                         break;
@@ -101,9 +136,19 @@
                 }
             } else {
                 hunks.add(hunk);
+                size += hunk.size();
+                lines += hunk.lines.size();
+            }
+
+            if (size > SIZE_LIMIT || lines > LINE_LIMIT) {
+                hunks = new SizeExceededHunks();
+                return hunks;
             }
         }
 
+        hunks.size = size;
+        hunks.lines = lines;
+
         return hunks;
     }
 
app/playRepository/GitRepository.java
--- app/playRepository/GitRepository.java
+++ app/playRepository/GitRepository.java
@@ -57,6 +57,10 @@
  */
 public class GitRepository implements PlayRepository {
 
+    public static final int DIFF_SIZE_LIMIT = 3 * FileDiff.SIZE_LIMIT;
+    public static final int DIFF_LINE_LIMIT = 3 * FileDiff.LINE_LIMIT;
+    public static final int DIFF_FILE_LIMIT = 2000;
+
     /**
      * Git 저장소 베이스 디렉토리
      */
@@ -1481,6 +1485,8 @@
         }
 
         List<FileDiff> result = new ArrayList<>();
+        int size = 0;
+        int lines = 0;
 
         for (DiffEntry diff : formatter.scan(treeParserA, treeParserB)) {
             FileDiff fileDiff = new FileDiff();
@@ -1495,26 +1501,47 @@
             String pathA = diff.getPath(DiffEntry.Side.OLD);
             String pathB = diff.getPath(DiffEntry.Side.NEW);
 
+            byte[] rawA = null;
             if (treeA != null
                     && Arrays.asList(DELETE, MODIFY, RENAME, COPY).contains(diff.getChangeType())) {
                 TreeWalk t1 = TreeWalk.forPath(repositoryA, pathA, treeA);
                 ObjectId blobA = t1.getObjectId(0);
-                byte[] rawA = repositoryA.open(blobA).getBytes();
+
+                try {
+                    rawA = repositoryA.open(blobA).getBytes();
+                } catch (org.eclipse.jgit.errors.LargeObjectException e) {
+                    result.add(fileDiff);
+                    continue;
+                }
 
                 fileDiff.isBinaryA = RawText.isBinary(rawA);
                 fileDiff.a = fileDiff.isBinaryA ? null : new RawText(rawA);
                 fileDiff.pathA = pathA;
             }
 
+            byte[] rawB = null;
             if (treeB != null
                     && Arrays.asList(ADD, MODIFY, RENAME, COPY).contains(diff.getChangeType())) {
                 TreeWalk t2 = TreeWalk.forPath(repositoryB, pathB, treeB);
                 ObjectId blobB = t2.getObjectId(0);
-                byte[] rawB = repositoryB.open(blobB).getBytes();
+
+                try {
+                    rawB = repositoryB.open(blobB).getBytes();
+                } catch (org.eclipse.jgit.errors.LargeObjectException e) {
+                    result.add(fileDiff);
+                    continue;
+                }
 
                 fileDiff.isBinaryB = RawText.isBinary(rawB);
                 fileDiff.b = fileDiff.isBinaryB ? null : new RawText(rawB);
                 fileDiff.pathB = pathB;
+            }
+
+            if (size > DIFF_SIZE_LIMIT || lines > DIFF_LINE_LIMIT) {
+                fileDiff.a = new RawText(new byte[0]);
+                fileDiff.b = new RawText(new byte[0]);
+                result.add(fileDiff);
+                continue;
             }
 
             if (!(fileDiff.isBinaryA || fileDiff.isBinaryB) && Arrays.asList(MODIFY, RENAME).contains(diff.getChangeType())) {
@@ -1525,6 +1552,22 @@
                                 DiffAlgorithm.SupportedAlgorithm.HISTOGRAM));
                 fileDiff.editList = diffAlgorithm.diff(RawTextComparator.DEFAULT, fileDiff.a,
                         fileDiff.b);
+                size += fileDiff.getHunks().size;
+                lines += fileDiff.getHunks().lines;
+            }
+
+            if (!fileDiff.isBinaryB && diff.getChangeType().equals(ADD)) {
+                lines += fileDiff.b.size();
+                size += rawB.length;
+            }
+
+            if (!fileDiff.isBinaryA && diff.getChangeType().equals(DELETE)) {
+                lines += fileDiff.a.size();
+                 size += rawA.length;
+            }
+
+            if (result.size() > DIFF_FILE_LIMIT) {
+                break;
             }
 
             result.add(fileDiff);
app/playRepository/Hunk.java
--- app/playRepository/Hunk.java
+++ app/playRepository/Hunk.java
@@ -13,4 +13,12 @@
     public int beginB;
     public int endB;
     public List<DiffLine> lines = new ArrayList<>();
+
+    public int size() {
+        int length = 0;
+        for (DiffLine line : lines) {
+            length += line.content.length();
+        }
+        return length;
+    }
 }
app/views/partial_diff.scala.html
--- app/views/partial_diff.scala.html
+++ app/views/partial_diff.scala.html
@@ -1,5 +1,9 @@
 @(fileDiffs: java.util.List[playRepository.FileDiff], comments:java.util.List[_ <: CodeComment] = new ArrayList[CodeComment], projectA: Project, projectB: Project)
 
+@if(fileDiffs.size >= playRepository.GitRepository.DIFF_FILE_LIMIT) {
+  <p class="alert">@Messages("code.fileDiffLimitExceeded", playRepository.GitRepository.DIFF_FILE_LIMIT)</p>
+}
+
 @for(diff <- fileDiffs) {
   @views.html.partial_filediff(diff, comments, projectA, projectB)
 }
app/views/partial_filediff.scala.html
--- app/views/partial_filediff.scala.html
+++ app/views/partial_filediff.scala.html
@@ -3,6 +3,7 @@
 @import playRepository.DiffLineType
 @import playRepository.DiffLine
 @import playRepository.FileDiff
+@import playRepository.Hunk
 @import org.eclipse.jgit.diff.DiffEntry
 @import org.eclipse.jgit.diff.RawText
 @import utils.TemplateHelper.DiffRenderer._
@@ -22,14 +23,19 @@
   @if(diff.isFileModeChanged) {
     <tr><td class="linenum"><div class="line-number" data-line-num="@diff.oldMode"></div><span class="hidden">@diff.oldMode</span></td><td class="linenum"><div class="line-number" data-line-num="@diff.newMode"></div><span class="hidden">@diff.newMode</span></td><td class="isBinary">@Messages("code.fileModeChanged")</td></tr>
   }
-  @if(diff.getHunks.size > 0){
-    @for(hunk <- diff.getHunks) {
-      <tr class="range"><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="hunk">@@@@ -@(hunk.beginA + 1),@(hunk.endA - hunk.beginA) +@(hunk.beginB + 1),@(hunk.endB - hunk.beginB) @@@@</td></tr>
-      @Html(renderLines(hunk.lines.toList, comments, eolMissingChecker(diff)))
+  @diff.getHunks match {
+    case hunks: FileDiff.SizeExceededHunks => { <tr><td colspan=3>@Messages("code.tooBigDiff")</td></tr> }
+    case hunks if (hunks.size <= 0) => {
+      @diff.isFileModeChanged match {
+        case true => { }
+        case false => { <tr><td colspan=3>@Messages("code.noChanges")</td></tr> }
+      }
     }
-  } else {
-    @if(!diff.isFileModeChanged) {
-      <tr><td colspan=3>@Messages("code.noChanges")</td></tr>
+    case hunks => {
+      @for(hunk <- diff.getHunks) {
+        <tr class="range"><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="hunk">@@@@ -@(hunk.beginA + 1),@(hunk.endA - hunk.beginA) +@(hunk.beginB + 1),@(hunk.endB - hunk.beginB) @@@@</td></tr>
+        @Html(renderLines(hunk.lines.toList, comments, eolMissingChecker(diff)))
+      }
     }
   }
 }
@@ -85,11 +91,14 @@
  * @param isBinaryOverwritten 기존 데이터가 바이너리인 경우 true
 **@
 @renderAddedLines(rawText: RawText, path: String, comments:Map[String, List[_ <: CodeComment]], isBinaryOverwritten: Boolean = false) = {
-  <tr class="range"><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="hunk">@@@@ -0,0 +1,@rawText.size @@@@</td></tr>
-  @if(isBinaryOverwritten ){
-      @renderCodeIsBinary("remove", "-")
+  @null match {
+    case _ if rawText.size == 0 => { <tr><td colspan=3>@Messages("code.skipDiff")</td></tr> }
+    case _ if FileDiff.isRawTextSizeExceeds(rawText) => { <tr><td colspan=3>@Messages("code.tooBigDiff")</td></tr> }
+    case _ => { <tr class="range"><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="hunk">@@@@ -0,0 +1,@rawText.size @@@@</td></tr>
+      @if(isBinaryOverwritten ) { @renderCodeIsBinary("remove", "-") }
+      @renderRawFile("add", path, rawText, CodeComment.Side.B, "+", comments)
+    }
   }
-  @renderRawFile("add", path, rawText, CodeComment.Side.B, "+", comments)
 }
 
 @**
@@ -101,10 +110,13 @@
  * @param isOverwrittenByBinary 새 데이터가 바이너리인 경우 true
 **@
 @renderRemovedLines(rawText: RawText, path: String, comments:Map[String, List[_ <: CodeComment]], isOverwrittenByBinary: Boolean = false) = {
-  <tr class="range"><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="hunk">@@@@ -1,@rawText.size +0,0 @@@@</td></tr>
-  @renderRawFile("remove", path, rawText, CodeComment.Side.A, "-", comments)
-  @if(isOverwrittenByBinary){
-      @renderCodeIsBinary("add", "+")      
+  @null match {
+    case _ if rawText.size == 0 => { <tr><td colspan=3>@Messages("code.skipDiff")</td></tr> }
+    case _ if FileDiff.isRawTextSizeExceeds(rawText) => { <tr><td colspan=3>@Messages("code.tooBigDiff")</td></tr> }
+    case _ => { <tr class="range"><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="linenum"><div class="line-number" data-line-num="..."><span class="hidden">...</span></div></td><td class="hunk">@@@@ -1,@rawText.size +0,0 @@@@</td></tr>
+      @renderRawFile("remove", path, rawText, CodeComment.Side.A, "-", comments)
+      @if(isOverwrittenByBinary) { @renderCodeIsBinary("add", "+") }
+    }
   }
 }
 
conf/messages
--- conf/messages
+++ conf/messages
@@ -80,6 +80,7 @@
 code.copyUrl = Copy URL
 code.deletedPath = {0} (Deleted)
 code.eolMissing = No newline at end of file
+code.fileDiffLimitExceeded = Up to {0} files are shown.
 code.fileModeChanged = File mode changed
 code.filename = Filename
 code.files = Files
@@ -105,6 +106,8 @@
 code.showCodeAtThisCommit = Browse the code at this point
 code.showCommit = View Commit
 code.showcomments = Show Comments
+code.skipDiff = This diff is skipped because others are too many.
+code.tooBigDiff = This diff is too big to show.
 common.attach.attachIfYouSave = will be attached when you save
 common.attach.clickToPost = Click to post
 common.attach.clickbutton = click Upload button
conf/messages.ko
--- conf/messages.ko
+++ conf/messages.ko
@@ -80,6 +80,7 @@
 code.copyUrl = 주소 복사
 code.deletedPath = {0} (삭제됨)
 code.eolMissing = 파일 끝에 줄바꿈 문자 없음
+code.fileDiffLimitExceeded = 최대 {0}개의 파일까지만 보여드립니다.
 code.fileModeChanged = 파일 모드 변경됨
 code.filename = 파일명
 code.files = 파일
@@ -105,6 +106,8 @@
 code.showCodeAtThisCommit = 이 시점의 파일내용 보기
 code.showCommit = 이 커밋 보기
 code.showcomments = 댓글 표시하기
+code.skipDiff = 다른 파일의 변경내역이 너무 많아서 이 파일의 변경내역은 생략합니다.
+code.tooBigDiff = 이 파일의 변경내역은 너무 커서 보여드릴 수 없습니다.
 common.attach.attachIfYouSave = 표시된 파일은 글을 저장하면 첨부됩니다.
 common.attach.clickToPost = 본문에 넣기
 common.attach.clickbutton = 버튼을 클릭해서 선택하세요
Add a comment
List