Skip to content
This repository was archived by the owner on Nov 11, 2022. It is now read-only.

Upgrade to Apache Beam, version 0.5.0 #546

Merged
merged 1 commit into from
Feb 2, 2017
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<version>3.5.1</version>
<configuration>
<source>${targetPlatform}</source>
<target>${targetPlatform}</target>
Expand Down Expand Up @@ -90,7 +90,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>2.18.1</version>
<version>2.19.1</version>
<configuration>
<parallel>all</parallel>
<threadCount>4</threadCount>
Expand All @@ -100,7 +100,7 @@
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-junit47</artifactId>
<version>2.18.1</version>
<version>2.19.1</version>
</dependency>
</dependencies>
</plugin>
Expand Down Expand Up @@ -201,7 +201,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.7</version>
<version>1.7.14</version>
</dependency>

<dependency>
Expand All @@ -223,7 +223,7 @@
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>4.11</version>
<version>4.12</version>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,9 @@ public FilterTextFn(String pattern) {
* in a dashboard, etc.
*/
private final Aggregator<Long, Long> matchedWords =
createAggregator("matchedWords", new Sum.SumLongFn());
createAggregator("matchedWords", Sum.ofLongs());
private final Aggregator<Long, Long> unmatchedWords =
createAggregator("umatchedWords", new Sum.SumLongFn());
createAggregator("unmatchedWords", Sum.ofLongs());

@ProcessElement
public void processElement(ProcessContext c) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ public class WordCount {
*/
static class ExtractWordsFn extends DoFn<String, String> {
private final Aggregator<Long, Long> emptyLines =
createAggregator("emptyLines", new Sum.SumLongFn());
createAggregator("emptyLines", Sum.ofLongs());

@ProcessElement
public void processElement(ProcessContext c) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
import ${package}.WordCount.CountWords;
import ${package}.WordCount.ExtractWordsFn;
import ${package}.WordCount.FormatAsTextFn;
import org.apache.beam.sdk.Pipeline;
import org.apache.beam.sdk.coders.StringUtf8Coder;
import org.apache.beam.sdk.testing.PAssert;
import org.apache.beam.sdk.testing.RunnableOnService;
Expand All @@ -34,6 +33,7 @@
import org.apache.beam.sdk.values.PCollection;
import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
Expand Down Expand Up @@ -68,12 +68,13 @@ public void testExtractWordsFn() throws Exception {
static final String[] COUNTS_ARRAY = new String[] {
"hi: 5", "there: 1", "sue: 2", "bob: 2"};

@Rule
public TestPipeline p = TestPipeline.create();

/** Example test that tests a PTransform by using an in-memory input and inspecting the output. */
@Test
@Category(RunnableOnService.class)
public void testCountWords() throws Exception {
Pipeline p = TestPipeline.create();

PCollection<String> input = p.apply(Create.of(WORDS).withCoder(StringUtf8Coder.of()));

PCollection<String> output = input.apply(new CountWords())
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<version>3.5.1</version>
<configuration>
<source>${targetPlatform}</source>
<target>${targetPlatform}</target>
Expand Down Expand Up @@ -81,12 +81,12 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.7</version>
<version>1.7.14</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>1.7.7</version>
<version>1.7.14</version>
</dependency>
</dependencies>
</project>
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<version>3.5.1</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
Expand Down Expand Up @@ -81,12 +81,12 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.7</version>
<version>1.7.14</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-jdk14</artifactId>
<version>1.7.7</version>
<version>1.7.14</version>
</dependency>
</dependencies>
</project>
28 changes: 27 additions & 1 deletion pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,32 @@
</repository>
</distributionManagement>

<repositories>
<repository>
<id>apache.staging</id>
<name>Apache Software Foundation Staging Repository</name>
<url>https://repository.apache.org/content/repositories/staging/</url>
<releases>
<enabled>true</enabled>
</releases>
<snapshots>
<enabled>false</enabled>
</snapshots>
</repository>

<repository>
<id>apache.snapshots</id>
<name>Apache Software Foundation Snapshot Repository</name>
<url>https://repository.apache.org/content/repositories/snapshots/</url>
<releases>
<enabled>false</enabled>
</releases>
<snapshots>
<enabled>true</enabled>
</snapshots>
</repository>
</repositories>

<prerequisites>
<maven>3.0.3</maven>
</prerequisites>
Expand All @@ -77,7 +103,7 @@
<timestamp>${maven.build.timestamp}</timestamp>
<maven.build.timestamp.format>yyyy-MM-dd HH:mm</maven.build.timestamp.format>

<beam.version>0.4.0</beam.version>
<beam.version>0.5.0</beam.version>

<dataflow.release_name>Google Cloud Dataflow SDK for Java</dataflow.release_name>
<dataflow.container_version>${project.version}-20170103</dataflow.container_version>
Expand Down