Skip to content
This repository has been archived by the owner on Jul 26, 2023. It is now read-only.

implementation of uniform data format for IDA #111

Open
wants to merge 48 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
e7a3a10
feat(rdf sparql server): Uniform data server initialized! with Spring…
Nov 17, 2018
4d228be
feat(rdf sparql server): check route added, XML2RDF added
Nov 18, 2018
5905044
feat(rdf sparql server): XML to RDF converter done
Nov 18, 2018
32fbb15
SParQL endpoint created, with output in JSON format
Nov 18, 2018
1805ca9
artist name query check and JSON format output on controller
Nov 18, 2018
0df6bae
reads all the query, removed the unnecessary things
Nov 18, 2018
d87c61f
Merge pull request #76 from dice-group/sparql-query-endpoint-2
maqboolkhan Nov 18, 2018
265e6ab
feat(rdf sparql server): gitignore update for IntelliJ
maqboolkhan Nov 18, 2018
6239e80
Merge pull request #77 from dice-group/basic-rdf-sparql-server
maqboolkhan Nov 18, 2018
1fb0e97
feat(rdf sparql server): Renamed files to Java standards
maqboolkhan Nov 19, 2018
0e61828
feat(rdf sparql server): csv to rdf added
maqboolkhan Nov 20, 2018
39b7a6d
Merge pull request #80 from dice-group/csv-adapter
maqboolkhan Nov 20, 2018
a1245e9
feat(rdf sparql server): XML file to RDF done
maqboolkhan Nov 20, 2018
db0e905
feat(rdf sparql server): CSV file to RDF done
maqboolkhan Nov 20, 2018
68c40ce
feat(rdf sparql server): created a util for handling file
maqboolkhan Nov 20, 2018
811dbbb
feat(rdf sparql server): Certain changes to make file upload more rel…
maqboolkhan Nov 20, 2018
80a0067
Merge pull request #82 from dice-group/file-upload-conversion
maqboolkhan Nov 20, 2018
07e7bc1
alternate approach for turtle datasets
AsjadLFC Dec 7, 2018
9693cd1
Merge pull request #96 from dice-group/sparql-alternate-approach
maqboolkhan Dec 7, 2018
0fce45c
code refactoring
maqboolkhan Dec 8, 2018
8623494
sentence added for chatbot rivescript
maqboolkhan Dec 8, 2018
51575e7
Added rivescript for dataset upload and actioncode so the UI can unde…
maqboolkhan Dec 8, 2018
32e4378
Reverting back everything
maqboolkhan Dec 8, 2018
95a4ec9
validation with file extension and file mime type added
maqboolkhan Dec 8, 2018
10497b0
Merge pull request #98 from dice-group/basic-validations
maqboolkhan Dec 8, 2018
e545580
Merge branch 'uniform-dataset-format' into integration-with-ida
maqboolkhan Dec 8, 2018
2a74fb1
response integration to/from UI for file
AsjadLFC Dec 11, 2018
b3e3cce
Modal dialog added for file upload
maqboolkhan Dec 15, 2018
bfeb849
Design added for file upload Modal dialog
maqboolkhan Dec 15, 2018
0ad70af
dataset is now uploading successfully
maqboolkhan Dec 15, 2018
eba297f
successfull and failed upload file reponse added
maqboolkhan Dec 16, 2018
8d2f003
code refactoring of file upload modal
maqboolkhan Dec 16, 2018
a1500c6
dataset upload set action code
AsjadLFC Dec 16, 2018
f0fb748
datset upload set action name
AsjadLFC Dec 16, 2018
3fb12e1
dataset name properly saving
maqboolkhan Dec 16, 2018
fe470d0
upload manager added
Dec 24, 2018
05f9edf
controller made for different files controlling for upload of files b…
Dec 31, 2018
1cc5e30
solved uploading files
Jan 9, 2019
30e2cce
datasets show, load, and query
Jan 13, 2019
c5f17dc
endpoint created, upload, show, query made
Jan 13, 2019
bf3e03a
Resolving conflicts
maqboolkhan Jan 15, 2019
4d3fa95
upload successfully without errors
Jan 17, 2019
eb02532
reposense sent back for fileMap
Jan 17, 2019
c3b2935
codacy issues resolve
Jan 21, 2019
cc4d46e
codacy isseus checking again
Jan 21, 2019
803ddf1
retesting codacy issues
Jan 21, 2019
a5c7bad
retesting codacy issues
Jan 21, 2019
feefefc
Merge branch 'master' into backend-implement-udf
Apr 17, 2019
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion ida-chatbot/src/app/app.component.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import {TabElement} from './models/tab-element';
import {UniqueIdProviderService} from './service/misc/unique-id-provider.service';
import {TabType} from './enums/tab-type.enum';
import {IdaEventService} from './service/event/ida-event.service';
import {MatDialog} from '@angular/material';
import {DatasetUploadModalComponent} from './components/dataset-upload-modal/dataset-upload-modal.component';

@Component({
selector: 'app-root',
Expand All @@ -31,7 +33,7 @@ export class AppComponent {
@ViewChild(SidebarComponent)
private sbComp: SidebarComponent;

constructor(private restservice: RestService, private uis: UniqueIdProviderService, private ies: IdaEventService) {
constructor(private restservice: RestService, private uis: UniqueIdProviderService, private ies: IdaEventService, public dialog: MatDialog) {
ies.dtTblEvnt.subscribe((reqTbl) => {
this.getDataTable(reqTbl);
});
Expand Down Expand Up @@ -65,6 +67,14 @@ export class AppComponent {
// Open new tab with DataTable
const newTab = new TabElement(this.uis.getUniqueId(), resp.payload.actvTbl, TabType.DTTBL, resp.payload.dataTable, true, true);
this.addNewTab(newTab, resp);
} else if (resp.actnCode === 6) {
this.dialog.open(DatasetUploadModalComponent, {
data: {
datasetName: resp.payload.datasetName
},
disableClose: true,
width: '350px'
});
} else if (resp.actnCode === 7) {
// Open new tab with DataTable
const newTab = new TabElement(this.uis.getUniqueId(), resp.payload.actvTbl, TabType.VENND, resp.payload.vennDiagramData, true, true);
Expand Down
8 changes: 6 additions & 2 deletions ida-chatbot/src/app/app.module.ts
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@ import {DatatableDetailComponent} from './components/datatable-detail/datatable-
import { SsbViewComponent } from './components/ssb-view/ssb-view.component';
import { VennViewComponent } from './components/venn-view/venn-view.component';
import { DeckglHexViewComponent } from './components/deckgl-hex-view/deckgl-hex-view.component';
import { DatasetUploadModalComponent } from './components/dataset-upload-modal/dataset-upload-modal.component';
@NgModule({
declarations: [
AppComponent,
Expand All @@ -78,7 +79,8 @@ import { DeckglHexViewComponent } from './components/deckgl-hex-view/deckgl-hex-
DatatableDetailComponent,
SsbViewComponent,
VennViewComponent,
DeckglHexViewComponent
DeckglHexViewComponent,
DatasetUploadModalComponent
],
imports: [
BrowserModule,
Expand Down Expand Up @@ -119,13 +121,15 @@ import { DeckglHexViewComponent } from './components/deckgl-hex-view/deckgl-hex-
MatTabsModule,
MatToolbarModule,
MatTooltipModule,
FlexLayoutModule
FlexLayoutModule,
MatSnackBarModule
],
providers: [{
provide: HTTP_INTERCEPTORS,
useClass: RestService ,
multi: true
}],
entryComponents: [DatasetUploadModalComponent],
bootstrap: [AppComponent]
})
export class AppModule { }
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
.file-name:disabled {
color: #000;
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
<h1 mat-dialog-title>Upload dataset</h1>
<div mat-dialog-content>
<form>
<mat-form-field class="full-width">
<input matInput class="file-name" [value]="fileName" name="fileName" disabled>
<mat-hint>You can only select CSV or XML file.</mat-hint>
</mat-form-field>
<br>
<br>
<button mat-stroked-button (click)="fileInput.click()" class="full-width">
<mat-icon>cloud_upload</mat-icon>
<input #fileInput type="file" [hidden]="true" accept=".xml, .csv" (change)="fileManager($event)"/>
</button>
</form>
</div>
<br>
<mat-dialog-actions align="end">
<button mat-button mat-dialog-close>Cancel</button>
<!-- The mat-dialog-close directive optionally accepts a value as a result for the dialog. -->
<button mat-stroked-button color="primary" [disabled]="!fileSelected || showProgress" (click)="uploadDataset()">
<mat-spinner *ngIf="showProgress" [diameter]="20" style="margin: auto"></mat-spinner>
<span *ngIf="!showProgress">Upload</span>
</button>
</mat-dialog-actions>
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
import { async, ComponentFixture, TestBed } from '@angular/core/testing';

import { DatasetUploadModalComponent } from './dataset-upload-modal.component';

describe('DatasetUploadModalComponent', () => {
let component: DatasetUploadModalComponent;
let fixture: ComponentFixture<DatasetUploadModalComponent>;

beforeEach(async(() => {
TestBed.configureTestingModule({
declarations: [ DatasetUploadModalComponent ]
})
.compileComponents();
}));

beforeEach(() => {
fixture = TestBed.createComponent(DatasetUploadModalComponent);
component = fixture.componentInstance;
fixture.detectChanges();
});

it('should create', () => {
expect(component).toBeTruthy();
});
});
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import {Component, Inject, Input, OnInit} from '@angular/core';
import {HttpClient} from '@angular/common/http';
import {MAT_DIALOG_DATA, MatDialogRef, MatSnackBar} from '@angular/material';

@Component({
selector: 'app-dialog-modal',
templateUrl: './dataset-upload-modal.component.html',
styleUrls: ['./dataset-upload-modal.component.css']
})
export class DatasetUploadModalComponent implements OnInit {

fileName = 'Select CSV or XML file...';
file;
fileSelected = false;
showProgress = false;

constructor(@Inject(MAT_DIALOG_DATA) public data,
private _http: HttpClient,
public dialogRef: MatDialogRef<DatasetUploadModalComponent>,
public snackBar: MatSnackBar) {}

ngOnInit() {
}

fileManager(event) {
if (event.target.files.length > 0) {
this.fileSelected = true;
this.file = event.target.files[0];
this.fileName = this.file.name;
}
}

uploadDataset() {
console.log(this.data.datasetName);
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Remove console.log

this.showProgress = true;
const formData = new FormData();
formData.append('file', this.file);
formData.append('fileName', this.data.datasetName);
this._http.post('http://localhost:8080/ida-ws/message/file', formData).subscribe(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This post method should be in some kind of service may be rest.service.ts

(res) => {
this.dialogRef.close();
this.snackBar.open('Dataset uploaded successfully', 'Close', {
duration: 3000,
});
},
(err) => {
this.snackBar.open('Dataset upload failed! Please try again', 'Close', {
duration: 3000,
});
this.showProgress = false;
}
);
}

}
4 changes: 3 additions & 1 deletion ida-chatbot/src/styles.css
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,9 @@ html, body {
height: 100%;
margin: 0;
}

.full-width {
width: 100%;
}
.table-cls {
margin: 15px;
}
Expand Down
12 changes: 11 additions & 1 deletion ida-ws/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -179,14 +179,24 @@
<artifactId>topicmodeling.commons</artifactId>
<version>0.0.3-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>org.apache.jena</groupId>
<artifactId>apache-jena-libs</artifactId>
<version>3.9.0</version>
<type>pom</type>
</dependency>
<dependency>
<groupId>no.acando</groupId>
<artifactId>xmltordf</artifactId>
<version>1.9.1</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-starter-test -->
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<version>2.1.0.RELEASE</version>
<scope>test</scope>
</dependency>

</dependencies>
<build>
<finalName>ida-ws</finalName>
Expand Down
5 changes: 4 additions & 1 deletion ida-ws/src/main/java/upb/ida/constant/IDALiteral.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@
public interface IDALiteral {
public static final String EXAMPLE = "this is an example constant string";

//Action literals
//Action literals (User Interface Action)
public static final int UIA_LOADDS = 1;
public static final int UIA_FDG = 2;
public static final int UIA_BG = 3;
public static final int UIA_CLUSTER = 4;
public static final int UIA_DTTABLE = 5;
public static final int UIA_UPLOAD = 6;
public static final int UIA_VENNDIAGRAM = 7;
public static final int UIA_GSDIAGRAM = 8;

Expand All @@ -37,4 +38,6 @@ public interface IDALiteral {
//Metadata File name Pattern
public static final String DSMD_FILE_PATTERN = ".*_dsmd\\.[jJ][sS][oO][nN]$";

public static final String DS_PATH = System.getProperty("user.dir") + "/uploads/";

}
15 changes: 14 additions & 1 deletion ida-ws/src/main/java/upb/ida/provider/LoadDsMetadata.java
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
package upb.ida.provider;

import java.io.IOException;
import java.util.HashMap;
import java.util.Map;

import org.apache.jena.rdf.model.Model;
import org.apache.jena.util.FileManager;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

Expand All @@ -12,6 +15,9 @@
import upb.ida.bean.ResponseBean;
import upb.ida.constant.IDALiteral;
import upb.ida.util.FileUtil;
import upb.ida.util.SessionUtil;

import static upb.ida.constant.IDALiteral.DS_PATH;

/**
* LoadDataContent is a subroutine that loads the data
Expand All @@ -25,7 +31,8 @@ public class LoadDsMetadata implements Subroutine {
private FileUtil fileUtil;
@Autowired
private ResponseBean responseBean;

@Autowired
private SessionUtil sessionUtil;
/**
* Method to create response for loading the data set
*
Expand All @@ -41,6 +48,12 @@ public String call(com.rivescript.RiveScript rs, String[] args) {
if (fileUtil.datasetExists(message)) {
try {
Map<String, Object> dataMap = responseBean.getPayload();
Model model = FileManager.get().loadModel(DS_PATH + message + ".ttl");
Map<String, Object> dsMap = new HashMap<>();
if(!dsMap.containsKey(message)){
dsMap.put(message, model);
sessionUtil.getSessionMap().put("DSModel", dsMap);
}
dataMap.put("label", message);
dataMap.put("dsName", message);
dataMap.put("dsMd", fileUtil.getDatasetMetaData(message));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ public class RiveScriptBeanProvider {
@Autowired
private LoadDsMetadata dsmdLoader;
@Autowired
private UploadDataset uploadDataset;
@Autowired
private ShowDataset showDataset;
@Autowired
private VennDiagramHandler VennDiagramHandler;
@Autowired
private GeoDiagramHandler GeoDiagramHandler;
Expand Down Expand Up @@ -77,6 +81,8 @@ public RiveScript initBotInstance() {
bot.setSubroutine("ClusterDataGetter", clusterDataGetter);
bot.setSubroutine("CheckParamCollected", checkParamCollected);
bot.setSubroutine("LoadDsMetadata", dsmdLoader);
bot.setSubroutine("UploadDataset", uploadDataset);
bot.setSubroutine("ShowDataset", showDataset);
bot.setSubroutine("VennDiagramHandler", VennDiagramHandler);
bot.setSubroutine("GeoDiagramHandler", GeoDiagramHandler);
return bot;
Expand Down
31 changes: 31 additions & 0 deletions ida-ws/src/main/java/upb/ida/provider/ShowDataset.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package upb.ida.provider;

import com.rivescript.macro.Subroutine;
import org.apache.commons.io.FilenameUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import upb.ida.bean.ResponseBean;

import java.io.File;

import static upb.ida.constant.IDALiteral.DS_PATH;


@Component
public class ShowDataset implements Subroutine {
@Autowired
private ResponseBean responseBean;

public String call(com.rivescript.RiveScript rs, String[] args) {
File directory = new File(DS_PATH);
String[] files = directory.list();
String temp = "Datasets: ";
for (String file: files) {
temp = temp + FilenameUtils.removeExtension(file) + ", ";
}

responseBean.setChatmsg(DS_PATH);
temp = temp.substring(0, temp.length() - 1);
return temp;
}
}
30 changes: 30 additions & 0 deletions ida-ws/src/main/java/upb/ida/provider/UploadDataset.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
package upb.ida.provider;

import com.rivescript.macro.Subroutine;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import upb.ida.bean.ResponseBean;
import upb.ida.constant.IDALiteral;

import java.io.File;
import java.util.Map;

@Component
public class UploadDataset implements Subroutine {
@Autowired
private ResponseBean responseBean;

public String call(com.rivescript.RiveScript rs, String[] args) {
String message = args[0].toLowerCase().trim();
Map<String, Object> dataMap = responseBean.getPayload();
dataMap.put("datasetName", message);
String resp = IDALiteral.RESP_FAIL_ROUTINE;
File file = new File(System.getProperty("user.dir") + "/upload-ds/" + message + ".ttl");
if (!file.exists()) {
responseBean.setActnCode(IDALiteral.UIA_UPLOAD);
responseBean.setPayload(dataMap);
resp = IDALiteral.RESP_PASS_ROUTINE;
}
return resp;
}
}
Loading