Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add metadata cmd #2091

Merged
merged 11 commits into from
Aug 26, 2022
Prev Previous commit
Next Next commit
Update javadocs
Signed-off-by: wslulciuc <[email protected]>
  • Loading branch information
wslulciuc committed Aug 26, 2022
commit 5dd275133bd8fd585871f2570401bdd8faa3d545
21 changes: 11 additions & 10 deletions api/src/main/java/marquez/cli/MetadataCommand.java
Original file line number Diff line number Diff line change
Expand Up @@ -41,18 +41,18 @@
*
* <h2>Usage</h2>
*
* For example, the following command will generate {@code metadata.json} with {@code 10} events,
* where each START event will have a size of {@code ~16384} bytes; events will be written to the
* {@code current} directory. You may specify the location of {@code metadata.json} by using the
* command-line argument {@code --output}
* For example, the following command will generate {@code metadata.json} with {@code 10} runs
* ({@code 20} events in total), where each START event will have a size of {@code ~16384} bytes;
* events will be written to {@code metadata.json} in the {@code current} directory. You may specify
* the location of {@code metadata.json} by using the command-line argument {@code --output}.
*
* <pre>{@code
* java -jar marquez-api.jar metadata --runs 10 --bytes-per-event 16384
* }</pre>
*/
@Slf4j
public final class MetadataCommand extends Command {
/* Used to calculate total bytes per event. */
/* Used to calculate (approximate) total bytes per event. */
private static final int BYTES_PER_RUN = 578;
private static final int BYTES_PER_JOB = 58;
private static final int BYTES_PER_FIELD_IN_SCHEMA = 256;
Expand All @@ -61,7 +61,7 @@ public final class MetadataCommand extends Command {
private static final int DEFAULT_NUM_OF_IO_PER_EVENT = 8;
private static final int DEFAULT_NUM_OF_FIELDS_IN_SCHEMA_PER_EVENT = 16;

/* Default limit. */
/* Default runs. */
private static final int DEFAULT_RUNS = 25;

/* Default bytes. */
Expand All @@ -79,6 +79,7 @@ public final class MetadataCommand extends Command {
private static final String CMD_ARG_METADATA_BYTES = "bytes-per-event";
private static final String CMD_ARG_METADATA_OUTPUT = "output";

/* Used for event randomization. */
private static final Random RANDOM = new Random();
private static final ZoneId AMERICA_LOS_ANGELES = ZoneId.of("America/Los_Angeles");
private static final List<String> FIELD_TYPES = ImmutableList.of("VARCHAR", "TEXT", "INTEGER");
Expand All @@ -103,7 +104,7 @@ public void configure(@NonNull Subparser subparser) {
.type(Integer.class)
.required(false)
.setDefault(DEFAULT_RUNS)
.help("limits runs up to N");
.help("limits OL runs up to N");
subparser
.addArgument("--bytes-per-event")
.dest("bytes-per-event")
Expand Down Expand Up @@ -205,8 +206,8 @@ private static RunEvents newOlRunEvents(final int bytesPerEvent) {

/** Write {@link OpenLineage.RunEvent}s to the specified {@code output}. */
private static void writeOlEvents(
@NonNull final List<OpenLineage.RunEvent> olEvents, @NonNull String output) {
System.out.format("Writing '%d' runs to: '%s'\n", olEvents.size() / 2, output);
@NonNull final List<OpenLineage.RunEvent> olEvents, @NonNull final String output) {
System.out.format("Writing '%d' events to: '%s'\n", olEvents.size(), output);
FileWriter fileWriter;
PrintWriter printWriter = null;
try {
Expand Down Expand Up @@ -362,6 +363,6 @@ private static int newId() {
return RANDOM.nextInt(Integer.MAX_VALUE - 1);
}

/** ... */
/** A container class for run info. */
record RunEvents(@NonNull OpenLineage.RunEvent start, @NonNull OpenLineage.RunEvent complete) {}
}