@article {56, title = {Multi-Objective Model Selection via Racing}, journal = {Cybernetics, IEEE Transactions on}, volume = {PP}, year = {2015}, pages = {1-1}, abstract = {Model selection is a core aspect in machine learning and is, occasionally, multi-objective in nature. For instance, hyper-parameter selection in a multi-task learning context is of multi-objective nature, since all the tasks{\textquoteright} objectives must be optimized simultaneously. In this paper, a novel multi-objective racing algorithm (RA), namely S-Race, is put forward. Given an ensemble of models, our task is to reliably identify Pareto optimal models evaluated against multiple objectives, while minimizing the total computational cost. As a RA, S-Race attempts to eliminate non-promising models with confidence as early as possible, so as to concentrate computational resources on promising ones. Simultaneously, it addresses the problem of multi-objective model selection (MOMS) in the sense of Pareto optimality. In S-Race, the nonparametric sign test is utilized for pair-wise dominance relationship identification. Moreover, a discrete Holm{\textquoteright}s step-down procedure is adopted to control the family-wise error rate of the set of hypotheses made simultaneously. The significance level assigned to each family is adjusted adaptively during the race. In order to illustrate its merits, S-Race is applied on three MOMS problems: 1) selecting support vector machines for classification; 2) tuning the parameters of artificial bee colony algorithms for numerical optimization; and 3) constructing optimal hybrid recommendation systems for movie recommendation. The experimental results confirm that S-Race is an efficient and effective MOMS algorithm compared to a brute-force approach.}, keywords = {Discrete Holm{\textquoteright}s step-down procedure, model selection, Multi-objective optimization, racing algorithm (RA), sign test (ST)}, issn = {2168-2267}, doi = {10.1109/TCYB.2015.2456187}, author = {Tiantian Zhang and Michael Georgiopoulos and Georgios C. Anagnostopoulos} } @article {48, title = {Multitask Classification Hypothesis Space With Improved Generalization Bounds}, journal = {Neural Networks and Learning Systems, IEEE Transactions on}, volume = {26}, year = {2015}, pages = {1468-1479}, abstract = {This paper presents a pair of hypothesis spaces (HSs) of vector-valued functions intended to be used in the context of multitask classification. While both are parameterized on the elements of reproducing kernel Hilbert spaces and impose a feature mapping that is common to all tasks, one of them assumes this mapping as fixed, while the more general one learns the mapping via multiple kernel learning. For these new HSs, empirical Rademacher complexity-based generalization bounds are derived, and are shown to be tighter than the bound of a particular HS, which has appeared recently in the literature, leading to improved performance. As a matter of fact, the latter HS is shown to be a special case of ours. Based on an equivalence to Group-Lasso type HSs, the proposed HSs are utilized toward corresponding support vector machine-based formulations. Finally, experimental results on multitask learning problems underline the quality of the derived and validate this paper{\textquoteright}s analysis.}, keywords = {Context, Hilbert space, Kernel, Learning systems, Machine learning, pattern recognition, statistical learning, supervised learning, Support vector machines, Training, Upper bound}, issn = {2162-237X}, doi = {10.1109/TNNLS.2014.2347054}, author = {Cong Li and Michael Georgiopoulos and Georgios C. Anagnostopoulos} } @conference {58, title = {Multi-Task Learning with Group-Specific Feature Space Sharing}, booktitle = {Machine Learning and Knowledge Discovery in Databases - ECML/PKDD 2015}, series = {Lecture Notes in Computer Science}, volume = {9285}, year = {2015}, note = {Acceptance rate 23.4\% (89/380).}, pages = {120-136}, publisher = {Springer International Publishing}, organization = {Springer International Publishing}, abstract = {When faced with learning a set of inter-related tasks from a limited amount of usable data, learning each task independently may lead to poor generalization performance. (MTL) exploits the latent relations between tasks and overcomes data scarcity limitations by co-learning all these tasks simultaneously to offer improved performance. We propose a novel Multi-Task Multiple Kernel Learning framework based on Support Vector Machines for binary classification tasks. By considering pair-wise task affinity in terms of similarity between a pair{\textquoteright}s respective feature spaces, the new framework, compared to other similar MTL approaches, offers a high degree of flexibility in determining how similar feature spaces should be, as well as which pairs of tasks should share a common feature space in order to benefit overall performance. The associated optimization problem is solved via a block coordinate descent, which employs a consensus-form Alternating Direction Method of Multipliers algorithm to optimize the Multiple Kernel Learning weights and, hence, to determine task affinities. Empirical evaluation on seven data sets exhibits a statistically significant improvement of our framework{\textquoteright}s results compared to the ones of several other Clustered Multi-Task Learning methods.}, doi = {10.1007/978-3-319-23525-7_8}, author = {Niloofar Yousefi and Michael Georgiopoulos and Georgios C. Anagnostopoulos} } @conference {40, title = {Multinomial Squared Direction Cosines Regression}, booktitle = {Neural Networks (IJCNN), The 2011 International Joint Conference on}, year = {2011}, note = {

Acceptance rate 75\% (468/620).

}, month = {July}, publisher = {Institute of Electrical and Electronics Engineers (IEEE)}, organization = {Institute of Electrical and Electronics Engineers (IEEE)}, address = {San Jose, California, USA }, abstract = {

In this paper we introduce Multinomial Squared Direction Cosines Regression as an alternative Multinomial Response Model. The proposed model offers an intuitive geometric interpretation to the task of estimating posterior class probabilities in multi-class problems. In specific, the latter probabilities correspond to the squared direction cosines between a given pattern and representative class exemplars that form a basis in the decision space. We demonstrate that the model allows for efficient training via a trust region based Newton{\textquoteright}s Method, provided that the number of model parameters is not too large. Experimental results on several benchmark classification problems show that it compares competitively against Logistic Regression Classifiers, Support Vector Machines, and Classification and Regression Tree models.

}, keywords = {benchmark classification problem, Computational modeling, decision space, decision theory, geometric interpretation, Kernel, logistic regression classifier, Mathematical model, model parameter, multiclass problem, multinomial response model, multinomial squared direction cosines regression, Newton method, pattern classification, regression analysis, Regression tree analysis, regression tree model, support vector machine, Support vector machines, Training}, doi = {10.1109/IJCNN.2011.6033620}, author = {Naveed H. Iqbal and Georgios C. Anagnostopoulos} } @conference {42, title = {Metric representations of data via the Kernel-based Sammon Mapping}, booktitle = {Neural Networks (IJCNN), The 2010 International Joint Conference on}, year = {2010}, note = {

Acceptance rate 71.5\% (621/868).

}, month = {July}, publisher = {Institute of Electrical and Electronics Engineers (IEEE)}, organization = {Institute of Electrical and Electronics Engineers (IEEE)}, address = {Barcelona, Spain}, abstract = {

In this paper we present a novel generalization of Sammon{\textquoteright}s Mapping (SM), which is a popular, metric multi-dimensional scaling technique used in data analysis and visualization. The new approach, namely the Kernel-based Sammon Mapping (KSM), yields the classic SM and other much related techniques as special cases. Apart from being able to approximate distance-preserving projections, it can also learn to metrically represent arbitrarily-defined dissimilarities or similarities between samples. Moreover, it can handle equally well numeric, categorical or mixed-type data. It is able to accomplish all this by modeling its projections as linear combinations of appropriate kernel functions. We report experimental results, which showcase KSM{\textquoteright}s capabilities in visually representing several meaningful relationships between samples of selected datasets.

}, keywords = {data analysis, data visualisation, data visualization, distance-preserving projections, kernel-based Sammon mapping, metric multidimensional scaling technique, metric representations, Visualization}, doi = {10.1109/IJCNN.2010.5596662}, author = {Mingbo Ma and Ryan Gonet and RuiZhi Yu and Georgios C. Anagnostopoulos} } @conference {43, title = {Multi-objective memetic evolution of ART-based classifiers}, booktitle = {Evolutionary Computation (CEC), 2010 IEEE Congress on}, year = {2010}, note = {

Nominated for Best Paper Award.

Acceptance rate 60\% (622/1027).

}, month = {July}, publisher = {Institute of Electrical and Electronics Engineers (IEEE)}, organization = {Institute of Electrical and Electronics Engineers (IEEE)}, address = {Barcelona, Spain}, abstract = {

In this paper we present a novel framework for evolving ART-based classification models, which we refer to as MOME-ART. The new training framework aims to evolve populations of ART classifiers to optimize both their classification error and their structural complexity. Towards this end, it combines the use of interacting sub-populations, some traditional elements of genetic algorithms to evolve these populations and a simulated annealing process used for solution refinement to eventually give rise to a multi-objective, memetic evolutionary framework. In order to demonstrate its capabilities, we utilize the new framework to train populations of semi-supervised Fuzzy ARTMAP and compare them with similar networks trained via the recently published MO-GART framework, which has been shown as being very effective in yielding high-quality ART-based classifiers. The experimental results show clear advantages of MOME-ART in terms of Pareto Front quality and density, as well as parsimony properties of the resulting classifiers.

}, keywords = {adaptive resonance theory, ART classifiers, ART-based classification models, ART-based classifiers, classification error, Complexity theory, Cooling, Electronic mail, fuzzy set theory, genetic algorithms, interacting subpopulations, Mathematical model, MO-GART framework, MOME-ART, multiobjective memetic evolutionary framework, Pareto front quality, Pareto optimisation, parsimony property, pattern classification, semisupervised fuzzy ARTMAP, simulated annealing, simulated annealing process, solution refinement, structural complexity, Subspace constraints, Training, training framework}, doi = {10.1109/CEC.2010.5586385}, author = {Rong Li and Timothy R. Mersch and Oriana X. Wen and Assem Kaylani and Georgios C. Anagnostopoulos} }