Merge pull request #40 from AlexRadch/Tests_Fails_On_Empty_Lines

Tests fails on empty lines

Author: Turnerj

src/TurnerSoftware.RobotsExclusionTools/Tokenization/TokenDefinition.cs

@@ -31,7 +31,7 @@ public TokenMatch Match(string input, int offset = 0)
 			}
 			else
 			{
-				return new TokenMatch { IsMatch = false };
+				return TokenMatch.NoMatch;
 			}
 		}
 	}

src/TurnerSoftware.RobotsExclusionTools/Tokenization/TokenMatch.cs

@@ -10,5 +10,7 @@ public class TokenMatch
 		public TokenType TokenType { get; set; }
 		public string Value { get; set; }
 		public int MatchLength { get; set; }
+
+		public static readonly TokenMatch NoMatch = new TokenMatch { IsMatch = false };
 	}
 }

src/TurnerSoftware.RobotsExclusionTools/Tokenization/TokenType.cs

@@ -12,6 +12,7 @@ public enum TokenType
 		Comment,
 		FieldValueDelimiter,
 		NewLine,
-		ValueDelimiter
+		ValueDelimiter,
+		Blank
 	}
 }

src/TurnerSoftware.RobotsExclusionTools/Tokenization/TokenizerBase.cs

@@ -52,13 +52,24 @@ private void Tokenize(string text, ICollection<Token> tokenCollection)
 			var offset = 0;
 			var numberOfChars = text.Length;
 
+			var lastTokenEnd = 0;
+
 			while (offset < numberOfChars)
 			{
 				var match = FindMatch(text, offset);
 				if (match.IsMatch)
 				{
+					if (lastTokenEnd != offset)
+					{
+						tokenCollection.Add(new Token(
+							TokenType.NotDefined,
+							text.Substring(lastTokenEnd, offset - lastTokenEnd)
+						));
+					}
+
 					tokenCollection.Add(new Token(match.TokenType, match.Value));
 					offset += match.MatchLength;
+					lastTokenEnd = offset;
 				}
 				else
 				{
@@ -78,7 +89,7 @@ private TokenMatch FindMatch(string text, int offset)
 				}
 			}
 
-			return new TokenMatch { IsMatch = false };
+			return TokenMatch.NoMatch;
 		}
 	}
 }

src/TurnerSoftware.RobotsExclusionTools/Tokenization/Tokenizers/RobotsFileTokenizer.cs

@@ -36,8 +36,9 @@ public class RobotsFileTokenizer : TokenizerBase
 			new TokenDefinition(TokenType.Comment, @"\G#[^\x0A\x0D]*"),
 			new TokenDefinition(TokenType.Field, @"\G[\x21\x23-\x27\x2a\x2b\x2d\x2e\x41-\x5a\x5e-\x7a\x7c\x7e]+(?=:[ ])"),
 			new TokenDefinition(TokenType.FieldValueDelimiter, @"\G:[ ]"),
-			new TokenDefinition(TokenType.Value, @"\G[^\x0A\x0D#]+"),
-			new TokenDefinition(TokenType.NewLine, @"\G\x0D?\x0A")
+			new TokenDefinition(TokenType.Value, @"\G(?<=:[ ])[^\x0A\x0D#]+"),
+			new TokenDefinition(TokenType.NewLine, @"\G\x0D?\x0A"),
+			new TokenDefinition(TokenType.Blank, @"\G[ \t]+")
 		};
 
 		protected override IEnumerable<TokenDefinition> GetTokenDefinitions()

tests/TurnerSoftware.RobotsExclusionTools.Tests/Resources/RobotsFile/Comprehensive-Example.txt

@@ -1,40 +1,41 @@
 User-agent: *
 Crawl-delay: 60
 Disallow: 
-
+ 
 Sitemap: http://www.example.org/sitemap.xml
-
+  		    
 User-agent: SinglePathAllowed
 Allow: /orgs/plan.html
 Disallow: /
-
+  
 User-agent: ImplicitWildcardAllowed
 Allow: /orgs
 Disallow: /
-
+  		   
 User-agent: ExplicitWildcardSuffix
 Allow: /org*
 Disallow: /
-
+  		
 User-agent: ExplicitWildcardPrefix
 Allow: *.html
 Disallow: /
-
+  		    
 User-agent: ContainedWildcard
 Allow: /org*/plan.html
 Disallow: /
 
 User-agent: PathWithAnyQueryString
 Disallow: /org/plan.html?
-
+  		        
 User-agent: PathWithPartQueryString
 Disallow: /org/plan.html?foo=bar*
-
+ 
 User-agent: PathMustStartWith
 Disallow: /org/plan.html
 
 User-agent: PathMustEndWith
 Disallow: /org/plan.html$
 Disallow: /org/planb.html*$
+  		    
+Sitemap: http://www.example.org/sitemap2.xml
 
-Sitemap: http://www.example.org/sitemap2.xml

tests/TurnerSoftware.RobotsExclusionTools.Tests/Resources/RobotsFile/FieldCasing-Example.txt

@@ -1,8 +1,9 @@
 User-agent: A
 Disallow: /RFCCompliant
-
+	
 User-Agent: B
 Disallow: /DifferentCaseUserAgent
-
+  		    
 User-agent: C
-DisAllow: /DifferentCaseRule
+DisAllow: /DifferentCaseRule
+     

tests/TurnerSoftware.RobotsExclusionTools.Tests/Resources/RobotsFile/NoRobots-RFC-Example.txt

@@ -1,16 +1,16 @@
 # /robots.txt for http://www.fict.org/
 # comments to [email protected]
-
 User-agent: unhipbot
 Disallow: /
-
+  		    
 User-agent: webcrawler
 User-agent: excite
 Disallow: 
-
+  	
 User-agent: *
 Disallow: /org/plans.html
 Allow: /org/
 Allow: /serv
 Allow: /~mak
-Disallow: /
+Disallow: /
+

tests/TurnerSoftware.RobotsExclusionTools.Tests/Resources/RobotsFile/NoUserAgentRules-Example.txt

@@ -1 +1,3 @@
-Disallow: /
+
+Disallow: /
+

tests/TurnerSoftware.RobotsExclusionTools.Tests/RobotsFile/RobotsFileTokenPatternValidatorTests.cs

@@ -38,9 +38,8 @@ public void MalformedFieldPatterns()
 			Assert.IsFalse(result.IsValid);
 
 			var firstErrorExpectedTokens = result.Errors.First().Expected;
-			Assert.AreEqual(TokenType.FieldValueDelimiter, firstErrorExpectedTokens.ElementAt(0));
-			Assert.AreEqual(TokenType.Field, firstErrorExpectedTokens.ElementAt(1));
-			Assert.AreEqual(2, firstErrorExpectedTokens.Count());
+			Assert.AreEqual(TokenType.NewLine, firstErrorExpectedTokens.ElementAt(0));
+			Assert.AreEqual(1, firstErrorExpectedTokens.Count());
 			Assert.AreEqual(19, result.Errors.Count());
 		}
 	}