RemoveHTML.scala
8e8f072aca5fmaster

RemoveHTML.scala
View Options

			/*
			* Warcbase: an open-source platform for managing web archives
			*
			* Licensed under the Apache License, Version 2.0 (the "License");
			* you may not use this file except in compliance with the License.
			* You may obtain a copy of the License at
			*
			* http://www.apache.org/licenses/LICENSE-2.0
			*
			* Unless required by applicable law or agreed to in writing, software
			* distributed under the License is distributed on an "AS IS" BASIS,
			* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
			* See the License for the specific language governing permissions and
			* limitations under the License.
			*/

			package org.warcbase.spark.matchbox

			import java.io.IOException

			import org.jsoup.Jsoup

			object RemoveHTML {
			def apply(content: String) = {
			try {
			Jsoup.parse(content).text().replaceAll("[\\r\\n]+", " ")
			}
			catch {
			case e: Exception => throw new IOException("Caught exception processing input row ", e)
			}
			}
			}